scale back objects to simply chunk user data. May add a more structur…

…ed object system in the future

FORMAT.md

@@ -34,8 +34,8 @@ Entities or some other extra data field needs to be added to chunks in the futur
 | Block Entities           | array[block entity] |                                                                                      |
 | Heightmap Mask           | int                 | A mask indicating which heightmaps are present. See `AnvilChunk` for flag constants. |
 | Heightmaps               | array[bytes]        | One heightmap for each bit present in Heightmap Mask                                 |
-| Number of Objects        | varint              | Number of entries in the following array                                             |
-| Objects                  | array[object]       |                                                                                      |
+| Length of user data      | varint              | Number of entries in the following array                                             |
+| User data                | array[byte]         |                                                                                      |
 
 ### Sections
 
@@ -64,16 +64,3 @@ Entities or some other extra data field needs to be added to chunks in the futur
 | Block Entity ID | string |                                      |
 | Has NBT Data    | bool   | If unset, NBT Data is omitted        |
 | NBT Data        | nbt    |                                      |
-
-### Object
-
-| Name                | Type        | Notes                                                            |
-|---------------------|-------------|------------------------------------------------------------------|
-| X                   | double      | World X position                                                 |
-| Y                   | double      | World Y position                                                 |
-| Z                   | double      | World Z position                                                 |
-| Yaw                 | float       |                                                                  |
-| Pitch               | float       |                                                                  |
-| Type                | string      | The type of the object. Completely up to the user implementation |
-| Length of user data | varint      | The length of the following array                                | 
-| User data           | array[byte] |                                                                  |

README.md

@@ -66,6 +66,36 @@ For example, to convert an anvil world while only selecting a 5 chunk radius aro
 AnvilPolar.anvilToPolar(Path.of("/path/to/anvil/world/dir"), ChunkSelector.radius(5));
 ```
 
+### User data & callbacks
+
+By default, Polar only stores blocks, biomes, block entities, and light data. However, in many cases it is desirable
+to have some additional user specific data stored in the world. To accommodate this use case, Polar chunks each have
+a "user data" field, which can contain any arbitrary data. To work with it, you must implement `PolarWorldAccess`,
+and provide an instance to the `PolarLoader` for example, the following will write the time of save in each chunks
+user data:
+
+```java
+public class UpdateTimeWorldAccess implements PolarWorldAccess {
+    private static final Logger logger = LoggerFactory.getLogger(UpdateTimeWorldAccess.class);
+
+    @Override
+    public void loadChunkData(@NotNull Chunk chunk, @Nullable NetworkBuffer userData) {
+        if (userData == null) return; // No saved data, probably first load
+
+        long lastSaveTime = userData.read(NetworkBuffer.LONG);
+        logger.info("loading chunk {}, {} which was saved at {}.", chunk.getChunkX(), chunk.getChunkZ(), lastSaveTime);
+    }
+
+    @Override
+    public void saveChunkData(@NotNull Chunk chunk, @NotNull NetworkBuffer userData) {
+        userData.write(NetworkBuffer.LONG, System.currentTimeMillis());
+    }
+}
+```
+
+Using a `PolarWorldAccess` implementation is as simple as attaching it to the `PolarLoader`: 
+`new PolarLoader(world).setWorldAccess(new UpdateTimeWorldAccess())`
+
 ## Comparison to others
 
 ### "Benchmark"

src/main/java/net/hollowcube/polar/AnvilPolar.java

@@ -241,12 +241,15 @@ public class AnvilPolar {
 //                OCEAN_FLOOR OCEAN_FLOOR_WG
 //                WORLD_SURFACE WORLD_SURFACE_WG
 
+                var userData = new byte[0];
+
                 chunks.add(new PolarChunk(
                         chunkReader.getChunkX(),
                         chunkReader.getChunkZ(),
                         sections,
                         blockEntities,
-                        heightmaps
+                        heightmaps,
+                        userData
                 ));
             }
         }

src/main/java/net/hollowcube/polar/PolarChunk.java

@@ -14,7 +14,8 @@ public record PolarChunk(
         int z,
         PolarSection[] sections,
         List<BlockEntity> blockEntities,
-        byte[][] heightmaps
+        byte[][] heightmaps,
+        byte[] userData
 ) {
 
     public static final int HEIGHTMAP_NONE = 0b0;

src/main/java/net/hollowcube/polar/PolarLoader.java

@@ -10,6 +10,7 @@
 import net.minestom.server.instance.*;
 import net.minestom.server.instance.block.Block;
 import net.minestom.server.instance.block.BlockManager;
+import net.minestom.server.network.NetworkBuffer;
 import net.minestom.server.utils.NamespaceID;
 import net.minestom.server.world.biomes.Biome;
 import net.minestom.server.world.biomes.BiomeManager;
@@ -22,6 +23,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
@@ -47,6 +49,7 @@ public class PolarLoader implements IChunkLoader {
     private final ReentrantReadWriteLock worldDataLock = new ReentrantReadWriteLock();
     private final PolarWorld worldData;
 
+    private PolarWorldAccess worldAccess = null;
     private boolean parallel = false;
 
     public PolarLoader(@NotNull Path path) throws IOException {
@@ -74,6 +77,12 @@ public PolarLoader(@NotNull PolarWorld world) {
         return worldData;
     }
 
+    @Contract("_ -> this")
+    public @NotNull PolarLoader setWorldAccess(@NotNull PolarWorldAccess worldAccess) {
+        this.worldAccess = worldAccess;
+        return this;
+    }
+
     /**
      * Sets the loader to save and load in parallel.
      * <br/><br/>
@@ -120,6 +129,7 @@ public void loadInstance(@NotNull Instance instance) {
         var chunk = CHUNK_SUPPLIER.createChunk(instance, chunkX, chunkZ);
         synchronized (chunk) {
             //todo replace with java locks, not synchronized
+            //   actually on second thought, do we really even need to lock the chunk? it is a local variable still
             int sectionY = chunk.getMinSection();
             for (var sectionData : chunkData.sections()) {
                 if (sectionData.isEmpty()) continue;
@@ -132,6 +142,11 @@ public void loadInstance(@NotNull Instance instance) {
             for (var blockEntity : chunkData.blockEntities()) {
                 loadBlockEntity(blockEntity, chunk);
             }
+
+            var userData = chunkData.userData();
+            if (userData.length > 0 && worldAccess != null) {
+                worldAccess.loadChunkData(chunk, new NetworkBuffer(ByteBuffer.wrap(userData)));
+            }
         }
 
         return CompletableFuture.completedFuture(chunk);
@@ -338,6 +353,9 @@ private void updateChunkData(@NotNull Short2ObjectMap<String> blockCache, @NotNu
         var heightmaps = new byte[32][PolarChunk.HEIGHTMAPS.length];
         //todo
 
+        byte[] userData = worldAccess == null ? new byte[0]
+                : NetworkBuffer.makeArray(b -> worldAccess.saveChunkData(chunk, b));
+
         worldDataLock.writeLock().lock();
         worldData.updateChunkAt(
                 chunk.getChunkX(),
@@ -347,15 +365,16 @@ private void updateChunkData(@NotNull Short2ObjectMap<String> blockCache, @NotNu
                         chunk.getChunkZ(),
                         sections,
                         blockEntities,
-                        heightmaps
+                        heightmaps,
+                        userData
                 )
         );
         worldDataLock.writeLock().unlock();
     }
 
     @Override
     public @NotNull CompletableFuture<Void> saveChunk(@NotNull Chunk chunk) {
-        throw new UnsupportedOperationException("Polar does not support saving individual chunks, see #saveChunks(Collection<Chunk>) or #saveInstance(Instance)");
+        return saveChunks(List.of(chunk));
     }
 
     private @NotNull String blockToString(@NotNull Block block) {

src/main/java/net/hollowcube/polar/PolarReader.java

@@ -62,17 +62,16 @@ private PolarReader() {}
         }
 
         // Objects
-        if (version > PolarWorld.VERSION_OBJECTS_OPT_BLOCK_ENT_NBT) {
-            var objectCount = buffer.read(VAR_INT);
-            Check.argCondition(objectCount != 0, "Objects are not supported yet!");
-            //todo objects
-        }
+        byte[] userData = new byte[0];
+        if (version > PolarWorld.VERSION_USERDATA_OPT_BLOCK_ENT_NBT)
+            userData = buffer.read(BYTE_ARRAY);
 
         return new PolarChunk(
                 chunkX, chunkZ,
                 sections,
                 blockEntities,
-                heightmaps
+                heightmaps,
+                userData
         );
     }
 
@@ -120,7 +119,7 @@ private PolarReader() {}
         var id = buffer.readOptional(STRING);
 
         NBTCompound nbt = null;
-        if (version <= PolarWorld.VERSION_OBJECTS_OPT_BLOCK_ENT_NBT || buffer.read(BOOLEAN))
+        if (version <= PolarWorld.VERSION_USERDATA_OPT_BLOCK_ENT_NBT || buffer.read(BOOLEAN))
             nbt = (NBTCompound) buffer.read(NBT);
 
         return new PolarChunk.BlockEntity(

src/main/java/net/hollowcube/polar/PolarWorld.java

@@ -18,7 +18,7 @@ public class PolarWorld {
     public static final short LATEST_VERSION = 3;
 
     static final short VERSION_UNIFIED_LIGHT = 1;
-    static final short VERSION_OBJECTS_OPT_BLOCK_ENT_NBT = 2;
+    static final short VERSION_USERDATA_OPT_BLOCK_ENT_NBT = 2;
 
     public static CompressionType DEFAULT_COMPRESSION = CompressionType.ZSTD;
 

src/main/java/net/hollowcube/polar/PolarWorldAccess.java

@@ -1,11 +1,38 @@
 package net.hollowcube.polar;
 
+import net.minestom.server.instance.Chunk;
+import net.minestom.server.network.NetworkBuffer;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
 /**
  * Provides access to user world data for a {@link PolarLoader} to get and set user
  * specific world data such as objects, as well as provides some relevant callbacks.
  * <br/><br/>
  * Usage if world access is completely optional, dependent features will not add
  * overhead to the format if unused.
  */
+@SuppressWarnings("UnstableApiUsage")
 public interface PolarWorldAccess {
+
+    /**
+     * Called when a chunk is created, just before it is added to the world.
+     * <br/><br/>
+     * Can be used to initialize the chunk based on saved user data in the world.
+     *
+     * @param chunk The Minestom chunk being created
+     * @param userData The saved user data, or null if none is present
+     */
+    default void loadChunkData(@NotNull Chunk chunk, @Nullable NetworkBuffer userData) {}
+
+    /**
+     * Called when a chunk is being saved.
+     * <br/><br/>
+     * Can be used to save user data in the chunk by writing it to the buffer.
+     *
+     * @param chunk The Minestom chunk being saved
+     * @param userData A buffer to write user data to save
+     */
+    default void saveChunkData(@NotNull Chunk chunk, @NotNull NetworkBuffer userData) {}
+
 }

src/main/java/net/hollowcube/polar/PolarWriter.java

@@ -50,8 +50,7 @@ private static void writeChunk(@NotNull NetworkBuffer buffer, @NotNull PolarChun
         //todo heightmaps
         buffer.write(INT, PolarChunk.HEIGHTMAP_NONE);
 
-        //todo objects
-        buffer.write(VAR_INT, 0);
+        buffer.write(BYTE_ARRAY, chunk.userData());
     }
 
     private static void writeSection(@NotNull NetworkBuffer buffer, @NotNull PolarSection section) {

src/test/java/net/hollowcube/polar/TestUserDataWriteRead.java

@@ -0,0 +1,45 @@
+package net.hollowcube.polar;
+
+import net.minestom.server.MinecraftServer;
+import net.minestom.server.instance.InstanceContainer;
+import net.minestom.server.network.NetworkBuffer;
+import net.minestom.server.world.DimensionType;
+import org.junit.jupiter.api.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class TestUserDataWriteRead {
+
+    static {
+        MinecraftServer.init();
+    }
+
+    @Test
+    void testWriteRead() {
+        var world = new PolarWorld();
+
+        var emptySections = new PolarSection[24];
+        Arrays.fill(emptySections, new PolarSection());
+        world.updateChunkAt(0, 0, new PolarChunk(0, 0, emptySections, List.of(), new byte[0][0], new byte[0]));
+
+        var wa = new UpdateTimeWorldAccess();
+        var loader = new PolarLoader(world).setWorldAccess(wa);
+        var instance = new InstanceContainer(UUID.randomUUID(), DimensionType.OVERWORLD, loader);
+        var chunk = loader.loadChunk(instance, 0, 0).join();
+
+        loader.saveChunk(chunk).join();
+
+        var newPolarChunk = world.chunkAt(0, 0);
+        var savedTime = new NetworkBuffer(ByteBuffer.wrap(newPolarChunk.userData())).read(NetworkBuffer.LONG);
+        assertEquals(wa.saveTime, savedTime);
+
+        loader.loadChunk(instance, 0, 0).join();
+        assertEquals(wa.loadTime, savedTime);
+    }
+
+}

src/test/java/net/hollowcube/polar/UpdateTimeWorldAccess.java

@@ -0,0 +1,31 @@
+package net.hollowcube.polar;
+
+import net.minestom.server.instance.Chunk;
+import net.minestom.server.network.NetworkBuffer;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("UnstableApiUsage")
+public class UpdateTimeWorldAccess implements PolarWorldAccess {
+    private static final Logger logger = LoggerFactory.getLogger(UpdateTimeWorldAccess.class);
+
+    public long saveTime = 0;
+    public long loadTime = 0;
+
+    @Override
+    public void loadChunkData(@NotNull Chunk chunk, @Nullable NetworkBuffer userData) {
+        if (userData == null) return; // No saved data, probably first load
+
+        long lastSaveTime = userData.read(NetworkBuffer.LONG);
+        logger.info("loading chunk {}, {} which was saved at {}.", chunk.getChunkX(), chunk.getChunkZ(), lastSaveTime);
+        loadTime = lastSaveTime;
+    }
+
+    @Override
+    public void saveChunkData(@NotNull Chunk chunk, @NotNull NetworkBuffer userData) {
+        saveTime = System.currentTimeMillis();
+        userData.write(NetworkBuffer.LONG, saveTime);
+    }
+}