Low memory, lazy and unload regions

CMakeLists.txt

@@ -17,12 +17,13 @@ if(CCACHE_FOUND)
 	set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
 endif(CCACHE_FOUND)
 
-set(ALL_LIBS
+set(LINKED_LIBS
 	${OPENGL_LIBRARY}
 	glfw
 	GLEW
 	pthread
 	dl
+	zstd
 )
 
 add_definitions(
@@ -35,18 +36,26 @@ add_definitions(
 	-mfma
 )
 
-file(GLOB_RECURSE SOURCES "src/*.cpp")
+file(GLOB_RECURSE SOURCES "src/*/*.cpp")
 file(GLOB INCLUDE_SOURCES "include/imgui-1.76/*.cpp" "include/FastNoiseSIMD/*.cpp" "include/Remotery/lib/*.c")
+set(INCLUDE_LIBS
+	"include/imgui-1.76"
+	"include/FastNoiseSIMD"
+	"include/toml++"
+	"include/Remotery/lib"
+	"include/robin_hood"
+)
 
-add_executable(univerxel ${SOURCES} ${INCLUDE_SOURCES})
+add_executable(univerxel "src/main.cpp" ${SOURCES} ${INCLUDE_SOURCES})
 target_compile_features(univerxel PUBLIC cxx_std_17)
-target_link_libraries(univerxel ${ALL_LIBS})
-target_include_directories(univerxel PRIVATE "include/imgui-1.76/" "include/FastNoiseSIMD/" "include/toml++/" "include/Remotery/lib/" "include/robin_hood")
+target_link_libraries(univerxel ${LINKED_LIBS})
+target_include_directories(univerxel PRIVATE ${INCLUDE_LIBS})
 if(PROFILING)
 	target_compile_definitions(univerxel PRIVATE RMT_ENABLED=1 RMT_USE_OPENGL=1)
 else(PROFILING)
 	target_compile_definitions(univerxel PRIVATE RMT_ENABLED=0)
 endif(PROFILING)
+add_dependencies(univerxel generate_dictionary)
 
 file(COPY content/shaders DESTINATION ${CMAKE_BINARY_DIR}/content)
 file(COPY content/textures DESTINATION ${CMAKE_BINARY_DIR}/content)
@@ -56,4 +65,17 @@ add_custom_target(docs
     COMMAND doxygen ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
     COMMENT "Build doc."
-)
+)
+
+# Zstd dictionary
+file(GLOB SMP_SOURCES "src/chunk_sampler.cpp" "src/world/Chunk.cpp" "include/FastNoiseSIMD/*.cpp")
+add_executable(chunk_sampler EXCLUDE_FROM_ALL ${SMP_SOURCES})
+target_compile_features(chunk_sampler PUBLIC cxx_std_17)
+target_link_libraries(chunk_sampler ${LINKED_LIBS})
+target_include_directories(chunk_sampler PRIVATE "include/FastNoiseSIMD")
+
+add_custom_command(OUTPUT "${CMAKE_BINARY_DIR}/content/zstd.dict"
+	COMMAND "${CMAKE_BINARY_DIR}/chunk_sampler"
+	COMMAND zstd --train "${CMAKE_BINARY_DIR}/samples/*" -o "${CMAKE_BINARY_DIR}/content/zstd.dict"
+	DEPENDS chunk_sampler)
+add_custom_target(generate_dictionary DEPENDS "${CMAKE_BINARY_DIR}/content/zstd.dict")

README.md

@@ -46,6 +46,7 @@ To get a local copy up and running, follow these simple steps.
 * OpenGL
 * GLFw
 * Glew
+* Zstd
 
 #### Optionally
 

TODO.md

@@ -5,7 +5,16 @@
 - [x] Density
 - [x] Robin hood map
 - [ ] Octree world
-- [ ] Serialize
+  - [ ] Remove density
+- [x] Serialize
+  - [ ] Variable length encoding
+  - [x] Group files
+    - [x] Zstd + custom grouping
+    - [~] Find best region size
+    - [x] Zstd Train dictionary
+    - [x] Low memory: Keep only ifstream
+    - [x] High memory: Save multiple
+    - [x] Unload unused
 - [ ] In memory RLE
 - [x] Edition
 - [ ] Entity
@@ -20,8 +29,15 @@
 - [ ] Leak test
   - Valgrind
     - Xtree-memory
+- [ ] sanitizer
+  - [ ] clang-tidy
+  - [ ] cmake
+  - https://github.com/microsoft/GSL/blob/master/include/gsl/pointers
 - [ ] Server
   - [ ] ZeroMQ
+- [ ] Mutex guard
+- [ ] Shared mutex
+- [ ] Logger
 
 ## Rendering
 - [x] Render triangle
@@ -57,4 +73,4 @@
 - [x] Frustum Culling
 - [ ] Occlusion Culling
 
-- [ ] Document
+- [x] Document

src/chunk_sampler.cpp

@@ -0,0 +1,32 @@
+/**
+ * \file chunk_sampler.cpp
+ * \brief Generate uncompressed chunks
+ * \author Maelys Bois
+ * \version 0.0.1
+ *
+ * Generate random uncompressed chunks for Zstd dictionary training.
+ */
+
+#include "world/Chunk.hpp"
+#include <cstdlib>
+#include <iostream>
+#include <fstream>
+#include <ctime>
+#include <filesystem>
+
+/// Entry point
+int main(int, char *[])
+{
+    std::srand(std::time(nullptr));
+    world::Generator generator(std::rand());
+    std::filesystem::create_directories("samples");
+    for (size_t i = 0; i < 10000; i++)
+    {
+        world::Chunk chunk(chunk_pos(std::rand(), std::rand(), std::rand()), generator);
+        std::ofstream out("samples/" + std::to_string(i));
+        chunk.write(out);
+        out.close();
+    }
+
+    return 0;
+}

src/contouring/Abstract.hpp

@@ -1,16 +1,11 @@
 #pragma once
 
-#include "../data/glm.hpp"
 #include "../render/buffer/Abstract.hpp"
 #include "../data/geometry/Frustum.hpp"
 #include "../data/geometry/Faces.hpp"
-#include <memory>
-#include <toml.h>
-#include <robin_hood.h>
+#include "../world/forward.h"
+typedef glm::vec3 camera_pos;
 
-namespace world {
-    class Chunk;
-}
 /// Mesh creation
 namespace contouring {
     /// Generating mesh from world data
@@ -24,10 +19,10 @@ namespace contouring {
         virtual void update(const camera_pos &pos) = 0;
 
         /// Chunk data change
-        virtual void onUpdate(const chunk_pos &pos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>>& data, geometry::Faces neighbors) = 0;
+        virtual void onUpdate(const chunk_pos &pos, const world::chunk_map& data, geometry::Faces neighbors) = 0;
         /// Chunk existante ping
         /// @note notify for chunks entering view while moving
-        virtual void onNotify(const chunk_pos &pos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &data) = 0;
+        virtual void onNotify(const chunk_pos &pos, const world::chunk_map &data) = 0;
         /// Display ImGui config
         virtual void onGui() = 0;
         /// Get options

src/contouring/AbstractFlat.cpp

@@ -1,6 +1,7 @@
 #include "AbstractFlat.hpp"
 
 #include <imgui.h>
+#include <toml.h>
 #include "../world/Chunk.hpp"
 
 namespace contouring {
@@ -42,7 +43,7 @@ namespace contouring {
         const auto scaling = glm::scale(glm::mat4(1), glm::vec3(scale));
         for (const auto [pos, buffer] : buffers) {
             if (buffer != NULL && (!frustum.has_value() || frustum.value().contains(geometry::Box::fromMin(scale * glm::vec3(pos) * glm::vec3(CHUNK_LENGTH), scale * glm::vec3(CHUNK_LENGTH)))))
-                out.push_back({glm::translate(scaling, glm::vec3(pos) * glm::vec3(CHUNK_LENGTH)), buffer});
+                out.emplace_back(glm::translate(scaling, glm::vec3(pos) * glm::vec3(CHUNK_LENGTH)), buffer);
         }
     }
 }

src/contouring/AbstractFlat.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "Abstract.hpp"
+#include "../data/math.hpp"
 
 namespace contouring {
     /// Generating mesh for chunks 1:1

src/contouring/Dummy.hpp

@@ -10,8 +10,8 @@ namespace contouring {
         virtual ~Dummy() { }
 
         void update(const camera_pos &) override { }
-        void onUpdate(const chunk_pos &, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &, geometry::Faces) override {}
-        void onNotify(const chunk_pos &, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &) override { }
+        void onUpdate(const chunk_pos &, const world::chunk_map &, geometry::Faces) override {}
+        void onNotify(const chunk_pos &, const world::chunk_map &) override { }
         void onGui() override { }
         std::string getOptions() override { return ""; }
         void getModels(std::vector<std::pair<glm::mat4, buffer::Abstract *const>> &, const std::optional<geometry::Frustum>&, float) override { }

src/contouring/FlatDualMC.cpp

@@ -4,6 +4,7 @@
 #include "../world/materials.hpp"
 #include <Remotery.h>
 #include <imgui.h>
+#include <toml.h>
 #include "dualmc.h"
 
 namespace contouring {
@@ -13,7 +14,7 @@ namespace contouring {
         manifold = opt["manifold"].value_or(manifold);
 
         for (size_t i = 1; i <= 2; i++) {
-            workers.push_back(std::thread([&] {
+            workers.emplace_back([&] {
                 while (running) {
                     std::pair<chunk_pos, surrounding::corners> ctx;
                     loadQueue.wait();
@@ -21,10 +22,10 @@ namespace contouring {
                         rmt_ScopedCPUSample(ProcessContouring, 0);
                         buffer::ShortIndexed::Data data;
                         render(ctx.second, data);
-                        loadedQueue.push({ctx.first, data});
+                        loadedQueue.emplace(ctx.first, data);
                     }
                 }
-            }));
+            });
         }
     }
     FlatDualMC::~FlatDualMC() {
@@ -48,7 +49,7 @@ namespace contouring {
         return ss.str();
     }
 
-     void FlatDualMC::enqueue(const chunk_pos &pos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &data) {
+     void FlatDualMC::enqueue(const chunk_pos &pos, const world::chunk_map &data) {
         rmt_ScopedCPUSample(EnqueueContouring, RMTSF_Aggregate);
         const auto dist2 = glm::length2(pos - center);
         if (dist2 <= loadDistance * loadDistance) {
@@ -59,7 +60,7 @@ namespace contouring {
         }
      }
 
-    void FlatDualMC::onUpdate(const chunk_pos &pos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &data, geometry::Faces neighbors) {
+    void FlatDualMC::onUpdate(const chunk_pos &pos, const world::chunk_map &data, geometry::Faces neighbors) {
         enqueue(pos, data);
         if (neighbors && (geometry::Faces::Left | geometry::Faces::Down | geometry::Faces::Backward)) {
             for (size_t i = 1; i < 8; i++) {
@@ -68,7 +69,7 @@ namespace contouring {
         }
     }
 
-    void FlatDualMC::onNotify(const chunk_pos &pos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &data) {
+    void FlatDualMC::onNotify(const chunk_pos &pos, const world::chunk_map &data) {
         if (buffers.find(pos) == buffers.end()) {
             enqueue(pos, data);
         }
@@ -109,15 +110,14 @@ namespace contouring {
     void FlatDualMC::render(const surrounding::corners &surrounding, buffer::ShortIndexed::Data &out) const {
         const int SIZE = CHUNK_LENGTH + 3;
         std::vector<dualmc::DualMC<float>::Point> grid;
-        grid.reserve(SIZE * SIZE * SIZE);
         {
+            grid.reserve(SIZE * SIZE * SIZE);
             for (int z = 0; z < SIZE; z++) {
             for (int y = 0; y < SIZE; y++) {
             for (int x = 0; x < SIZE; x++) {
-                const auto chunk = surrounding[(z >= CHUNK_LENGTH) + (y >= CHUNK_LENGTH)*2 + (x >= CHUNK_LENGTH)*4];
-                // MAYBE: area copy
-                const auto voxel = chunk->getAt(chunk_voxel_pos(x % CHUNK_LENGTH, y % CHUNK_LENGTH, z % CHUNK_LENGTH));
-                grid.push_back({voxel.Density * 1.f / UCHAR_MAX, voxel.Material});
+                const auto &chunk = surrounding[(z >= CHUNK_LENGTH) + (y >= CHUNK_LENGTH)*2 + (x >= CHUNK_LENGTH)*4];
+                const auto &voxel = chunk->getAt(chunk_voxel_pos(x % CHUNK_LENGTH, y % CHUNK_LENGTH, z % CHUNK_LENGTH));
+                grid.emplace_back(voxel.Density * 1.f / UCHAR_MAX, voxel.Material);
             }}}
         }
         {
@@ -131,9 +131,9 @@ namespace contouring {
             out.materials.reserve(dmc_vertices.size());
             out.normals.reserve(dmc_vertices.size());
             for (const auto& v: dmc_vertices) {
-                out.vertices.push_back(glm::vec3(v.x, v.y, v.z));
-                out.materials.push_back(v.w);
-                out.normals.push_back(glm::vec3(0));
+                out.vertices.emplace_back(v.x, v.y, v.z);
+                out.materials.emplace_back(v.w);
+                out.normals.emplace_back(0);
             }
 
             out.indices.reserve(dmc_tris.size());

src/contouring/FlatDualMC.hpp

@@ -26,10 +26,10 @@ namespace contouring {
         std::string getOptions() override;
 
         /// Chunk data change
-        void onUpdate(const chunk_pos &, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &, geometry::Faces) override;
+        void onUpdate(const chunk_pos &, const world::chunk_map &, geometry::Faces) override;
         /// Chunk existante ping
         /// @note notify for chunks entering view while moving
-        void onNotify(const chunk_pos &, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &) override;
+        void onNotify(const chunk_pos &, const world::chunk_map &) override;
 
     protected:
         safe_priority_queue_map<chunk_pos, surrounding::corners, int> loadQueue;
@@ -44,7 +44,7 @@ namespace contouring {
         bool running = true;
         std::vector<std::thread> workers;
 
-        void enqueue(const chunk_pos &, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &);
+        void enqueue(const chunk_pos &, const world::chunk_map &);
 
         float iso = .1f;
         bool manifold = true;

src/contouring/FlatSurroundingBox.cpp

@@ -9,7 +9,7 @@ using namespace geometry;
 namespace contouring {
     FlatSurroundingBox::FlatSurroundingBox(const std::string &opt) : AbstractFlat(opt) {
         for (size_t i = 1; i <= 4; i++) {
-            workers.push_back(std::thread([&] {
+            workers.emplace_back([&] {
                 while (running) {
                     std::pair<chunk_pos, surrounding::faces> ctx;
                     loadQueue.wait();
@@ -19,11 +19,11 @@ namespace contouring {
                         render(ctx.second, vertices);
                         {
                             rmt_ScopedCPUSample(Index, 0);
-                            loadedQueue.push({ctx.first, buffer::ShortIndexed::Data(vertices)});
+                            loadedQueue.emplace(ctx.first, vertices);
                         }
                     }
                 }
-            }));
+            });
         }
     }
     FlatSurroundingBox::~FlatSurroundingBox() {
@@ -36,7 +36,7 @@ namespace contouring {
         }
     }
 
-     void FlatSurroundingBox::enqueue(const chunk_pos &pos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &data) {
+     void FlatSurroundingBox::enqueue(const chunk_pos &pos, const world::chunk_map &data) {
         rmt_ScopedCPUSample(EnqueueContouring, RMTSF_Aggregate);
         const auto dist2 = glm::length2(pos - center);
         if (dist2 <= loadDistance * loadDistance) {
@@ -47,7 +47,7 @@ namespace contouring {
         }
      }
 
-    void FlatSurroundingBox::onUpdate(const chunk_pos &pos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &data, Faces neighbors) {
+    void FlatSurroundingBox::onUpdate(const chunk_pos &pos, const world::chunk_map &data, Faces neighbors) {
         enqueue(pos, data);
         if (neighbors && Faces::Right)
             enqueue(pos + g_face_offsets[static_cast<int>(Face::Right)], data);
@@ -68,7 +68,7 @@ namespace contouring {
             enqueue(pos + g_face_offsets[static_cast<int>(Face::Backward)], data);
     }
 
-    void FlatSurroundingBox::onNotify(const chunk_pos &pos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &data) {
+    void FlatSurroundingBox::onNotify(const chunk_pos &pos, const world::chunk_map &data) {
         if (buffers.find(pos) == buffers.end()) {
             enqueue(pos, data);
         }

src/contouring/FlatSurroundingBox.hpp

@@ -24,10 +24,10 @@ namespace contouring {
         void onGui() override;
 
         /// Chunk data change
-        void onUpdate(const chunk_pos &, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &, geometry::Faces) override;
+        void onUpdate(const chunk_pos &, const world::chunk_map &, geometry::Faces) override;
         /// Chunk existante ping
         /// @note notify for chunks entering view while moving
-        void onNotify(const chunk_pos &, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &) override;
+        void onNotify(const chunk_pos &, const world::chunk_map &) override;
 
     protected:
         safe_priority_queue_map<chunk_pos, surrounding::faces, int> loadQueue;
@@ -42,7 +42,7 @@ namespace contouring {
         bool running = true;
         std::vector<std::thread> workers;
 
-        void enqueue(const chunk_pos &, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &);
+        void enqueue(const chunk_pos &, const world::chunk_map &);
 
     private:
         static inline bool isTransparent(const surrounding::faces &surrounding, const std::pair<ushort, ushort> &idx);

src/contouring/boxing.hpp

@@ -27,7 +27,7 @@ namespace contouring::box {
 
     static void addQuad(std::vector<buffer::VertexData> & out, glm::vec3 position, ushort material, Face face, glm::vec3 size) {
             for (auto vertex : g_quad_vertices) {
-                out.push_back(buffer::VertexData{glm::vec3(g_cube_rotate[static_cast<int>(face)] * glm::vec4(vertex, 1)) * size + position, material, g_cube_normals[static_cast<int>(face)]});
+                out.emplace_back(glm::vec3(g_cube_rotate[static_cast<int>(face)] * glm::vec4(vertex, 1)) * size + position, material, g_cube_normals[static_cast<int>(face)]);
             }
         }
 

src/contouring/index.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "Abstract.hpp"
+#include <map>
 
 namespace contouring {
     static const std::array<std::string, 3> names = {"FlatDualMC", "FlatBox", "Dummy"};

src/contouring/surrounding.cpp

@@ -1,10 +1,11 @@
 #include "surrounding.hpp"
 
 #include "../world/Chunk.hpp"
+#include "../data/math.hpp"
 
 using namespace geometry;
 namespace contouring::surrounding {
-    bool load(faces &out, const chunk_pos &chunkPos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &chunks) {
+    bool load(faces &out, const chunk_pos &chunkPos, const world::chunk_map &chunks) {
         {
             const auto it = chunks.find(chunkPos);
             if (it == chunks.end())
@@ -59,7 +60,7 @@ namespace contouring::surrounding {
         }
     }
 
-    bool load(corners &out, const chunk_pos &chunkPos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &chunks) {
+    bool load(corners &out, const chunk_pos &chunkPos, const world::chunk_map &chunks) {
         for (size_t i = 0; i < 8; i++) {
             const auto it = chunks.find(chunkPos + g_corner_offsets[i]);
             if (it == chunks.end())

src/contouring/surrounding.hpp

@@ -1,8 +1,7 @@
 #pragma once
 
 #include "../data/geometry/Faces.hpp"
-#include <memory>
-#include <robin_hood.h>
+#include "../world/forward.h"
 
 namespace world {
     class Chunk;
@@ -11,7 +10,7 @@ namespace contouring::surrounding {
     const auto CENTER = 6;
     typedef std::array<std::shared_ptr<const world::Chunk>, CENTER+1> faces;
 
-    bool load(faces &out, const chunk_pos &chunkPos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &chunks);
+    bool load(faces &out, const chunk_pos &chunkPos, const world::chunk_map &chunks);
 
     std::pair<ushort, ushort> getNeighborIdx(ushort idx, geometry::Face face);
 
@@ -27,5 +26,5 @@ namespace contouring::surrounding {
         glm::ivec3(1, 1, 1),
     };
 
-    bool load(corners &out, const chunk_pos &chunkPos, const robin_hood::unordered_map<chunk_pos, std::shared_ptr<world::Chunk>> &chunks);
+    bool load(corners &out, const chunk_pos &chunkPos, const world::chunk_map &chunks);
 }

src/control/Camera.hpp

@@ -7,6 +7,7 @@
 #include "../data/glm.hpp"
 #include "../data/geometry/Frustum.hpp"
 #include "../data/geometry/Ray.hpp"
+typedef glm::vec3 camera_pos;
 
 /// Moving perspective camera
 class Camera {
@@ -37,10 +38,10 @@ public:
     inline geometry::Frustum getFrustum() const { return geometry::Frustum(ViewMatrix, ProjectionMatrix); }
     inline geometry::Ray getRay() const { return geometry::Ray(Position, getDirection(), o.far); }
 
-    glm::mat4 getViewMatrix() const { return ViewMatrix; }
-    glm::mat4 getProjectionMatrix() const { return ProjectionMatrix; }
-    camera_pos getPosition() const { return Position; }
-    float getDepth() const { return o.far; }
+    constexpr glm::mat4 getViewMatrix() const { return ViewMatrix; }
+    constexpr glm::mat4 getProjectionMatrix() const { return ProjectionMatrix; }
+    constexpr camera_pos getPosition() const { return Position; }
+    constexpr float getDepth() const { return o.far; }
 
 private:
     GLFWwindow *window;

src/data/geometry/Ray.hpp

@@ -19,12 +19,12 @@ namespace geometry {
 
         /// Get path points in integer grid
         /// @note not precise enough
-        inline void grid(std::vector<voxel_pos>& points) const {
-            voxel_pos current = from + glm::vec3(.5f);
-            voxel_pos d = dir * dist;
-            voxel_pos inc = voxel_pos((d.x < 0) ? -1 : 1, (d.y < 0) ? -1 : 1, (d.z < 0) ? -1 : 1);
-            voxel_pos size = glm::abs(d);
-            voxel_pos delta = size << 1ll;
+        inline void grid(std::vector<glm::lvec3>& points) const {
+            glm::lvec3 current = from + glm::vec3(.5f);
+            const glm::lvec3 d = dir * dist;
+            const glm::lvec3 inc = glm::lvec3((d.x < 0) ? -1 : 1, (d.y < 0) ? -1 : 1, (d.z < 0) ? -1 : 1);
+            const glm::lvec3 size = glm::abs(d);
+            const glm::lvec3 delta = size << 1ll;
 
             if ((size.x >= size.y) && (size.x >= size.z)) {
                 int err_1 = delta.y - size.x;

src/data/geometry/Sphere.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <glm/glm.hpp>
+#include <vector>
+
+/// Interger sphere fill
+struct SphereIterator {
+    SphereIterator(const glm::ivec3 &center, int radius): center(center), radius(radius) { }
+
+    glm::ivec3 center;
+    int radius;
+
+    void vector(std::vector<glm::ivec3>& out) {
+        int top = center.y - radius, bottom = center.y + radius;
+
+        for (int y = top; y <= bottom; y++) {
+            int dy = y - center.y, dxz = floor(sqrt(radius * radius - dy * dy));
+            int minx = center.x - dxz, maxx = center.x + dxz;
+            int minz = center.z - dxz, maxz = center.z + dxz;
+            out.reserve(out.size() + dxz * dxz);
+            for (int z = minz; z <= maxz; z++) {
+            for (int x = minx; x <= maxx; x++) {
+                out.emplace_back(x, y, z);
+            }}
+        }
+    }
+};

src/data/glm.hpp

@@ -1,40 +1,10 @@
 #pragma once
 
 #include <glm/glm.hpp>
-#include <glm/gtx/hash.hpp>
 
 namespace glm {
     typedef vec<3, long long> lvec3;
     typedef vec<4, long long> lvec4;
-}
-
-typedef glm::vec3 camera_pos;
-typedef glm::lvec3 voxel_pos;
-typedef glm::ivec3 chunk_pos;
-typedef glm::vec<3, ushort> chunk_voxel_pos;
-
-namespace glm {
-    ivec3 inline iround(const vec3& p) {
-        return ivec3(std::round<int>(p.x), std::round<int>(p.y), std::round<int>(p.z));
-    }
-    int inline length2(const ivec3& a) {
-        return a.x * a.x + a.y * a.y + a.z * a.z;
-    }
-    ivec3 inline diff(const ivec3& a, const ivec3& b) {
-        return glm::abs(glm::abs(a) - glm::abs(b));
-    }
-
-    uint inline rem(long long value, uint m) {
-        return value < 0 ? ((value+1) % (long long)m) + m - 1 : value % (long long)m;
-    }
-    int inline div(long long value, uint m) {
-        return value < 0 ? ((value+1) / (long long)m) - 1 : value / (long long)m;
-    }
-    chunk_voxel_pos inline modulo(const voxel_pos& value, const chunk_voxel_pos& m) {
-        return chunk_voxel_pos(rem(value.x, m.x), rem(value.y, m.y), rem(value.z, m.z));
-    }
-    chunk_pos inline divide(const voxel_pos &value, const chunk_voxel_pos &m)
-    {
-        return chunk_pos(div(value.x, m.x), div(value.y, m.y), div(value.z, m.z));
-    }
+    typedef vec<3, ushort> usvec3;
+    typedef vec<3, unsigned char> ucvec3;
 }

src/data/math.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "glm.hpp"
+#include <glm/gtx/hash.hpp>
+
+namespace glm {
+    constexpr ivec3 inline iround(const vec3& p) {
+        return ivec3(std::round<int>(p.x), std::round<int>(p.y), std::round<int>(p.z));
+    }
+    constexpr int inline length2(const ivec3& a) {
+        return a.x * a.x + a.y * a.y + a.z * a.z;
+    }
+    constexpr ivec3 inline diff(const ivec3& a, const ivec3& b) {
+        return glm::abs(glm::abs(a) - glm::abs(b));
+    }
+
+    constexpr uint inline rem(long long value, uint m) {
+        return value < 0 ? ((value+1) % (long long)m) + m - 1 : value % (long long)m;
+    }
+    constexpr int inline div(long long value, uint m) {
+        return value < 0 ? ((value+1) / (long long)m) - 1 : value / (long long)m;
+    }
+    constexpr ucvec3 inline modulo(const lvec3& value, const ucvec3& m) {
+        return ucvec3(rem(value.x, m.x), rem(value.y, m.y), rem(value.z, m.z));
+    }
+    constexpr ivec3 inline divide(const lvec3 &value, const ucvec3 &m) {
+        return ivec3(div(value.x, m.x), div(value.y, m.y), div(value.z, m.z));
+    }
+    constexpr std::pair<ivec3, ucvec3> inline split(const lvec3 &value, const ucvec3 &m) {
+        return {divide(value, m), modulo(value, m)};
+    }
+}

src/data/safe_priority_queue.hpp

@@ -25,7 +25,7 @@ namespace data {
     public:
         void push(const K& key, const V& val, const W& weight) {
             std::unique_lock<std::mutex> lock(mutex);
-            heap.push_back({key, weight});
+            heap.emplace_back(key, weight);
             std::push_heap(heap.begin(), heap.end(), cmpByWeight);
             map.insert_or_assign(key, val);
             cv.notify_one();
@@ -80,14 +80,14 @@ namespace data {
         }
 
         std::vector<std::pair<K, W>> heap;
-        robin_hood::unordered_set<K> set;
+        robin_hood::unordered_flat_set<K> set;
         std::mutex mutex;
         std::condition_variable cv;
 
     public:
         void push(const K& key, const W& weight) {
             std::unique_lock<std::mutex> lock(mutex);
-            heap.push_back({key, weight});
+            heap.emplace_back(key, weight);
             std::push_heap(heap.begin(), heap.end(), cmpByWeight);
             set.insert(key);
             cv.notify_one();

src/data/safe_queue.hpp

@@ -3,6 +3,7 @@
 #include <queue>
 #include <mutex>
 #include <condition_variable>
+#include <robin_hood.h>
 
 namespace data {
     /// Thread safe queue
@@ -20,6 +21,13 @@ namespace data {
             cv.notify_one();
         }
 
+        template <typename... _Args>
+        void emplace(_Args &&... __args) {
+            std::unique_lock<std::mutex> lock(mutex);
+            queue.emplace(std::forward<_Args>(__args)...);
+            cv.notify_one();
+        }
+
         bool pop(T& out) {
             std::unique_lock<std::mutex> lock(mutex);
             if (queue.empty())

src/data/safe_unique_queue.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <queue>
-#include <unordered_set>
+#include <robin_hood.h>
 #include <mutex>
 #include <condition_variable>
 
@@ -11,7 +11,7 @@ namespace data {
     class safe_unique_queue {
     private:
         std::queue<T> queue;
-        std::unordered_set<T> set;
+        robin_hood::unordered_flat_set<T> set;
         std::mutex mutex;
         std::condition_variable cv;
 

src/data/unique_queue.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <queue>
-#include <unordered_set>
+#include <robin_hood.h>
 #include <cassert>
 
 namespace data {
@@ -10,7 +10,7 @@ namespace data {
     template <class T>
     struct unique_queue {
         std::queue<T> queue;
-        std::unordered_set<T> set;
+        robin_hood::unordered_flat_set<T> set;
 
         bool push(T in) {
             if(set.insert(in).second) {

src/main.cpp

@@ -7,22 +7,18 @@
  * Univerxel main program.
  */
 
-#include <stdexcept>
 #include <iostream>
 #include <chrono>
-#include <thread>
 
 #include "render/window.hpp"
 #include "render/UI.hpp"
-#include "control/InputMap.hpp"
-#include "control/Camera.hpp"
+#include "control/InputMap.hpp" //TODO: add options
 
-#include "render/Renderer.hpp"
 #include "render/pass/ColorProgram.hpp"
 #include "render/buffer/Colored.hpp"
-#include "world/Universe.hpp"
 
 #include "state.h"
+#include "data/math.hpp"
 
 #include <Remotery.h>
 

src/render/Renderer.cpp

@@ -49,7 +49,7 @@ void Renderer::unloadTextures() {
 void Renderer::loadTextures(const std::string& texturePath, float mipMapLOD) {
     std::vector<std::string> terrainTextures;
     for(const auto texture: world::materials::textures) {
-        terrainTextures.push_back(texturePath + "/terrain/" + texture);
+        terrainTextures.emplace_back(texturePath + "/terrain/" + texture);
     }
     TextureAtlas = pass::Program::loadTextureArray(terrainTextures, "", mipMapLOD);
     NormalAtlas = pass::Program::loadTextureArray(terrainTextures, ".nrm", mipMapLOD);

src/render/Renderer.hpp

@@ -1,10 +1,9 @@
 #pragma once
 
 #include <GL/glew.h>
-#include <imgui.h>
+#include "pass/Context.hpp"
 #include "pass/MainProgram.hpp"
 #include "pass/SkyProgram.hpp"
-#include "pass/Context.hpp"
 
 class Camera;
 /// Handle rendering passes and params
@@ -23,7 +22,7 @@ public:
         /// Textures quality
         float mipMapLOD = -.5;
         /// Depth color
-        ImVec4 clear_color;
+        glm::vec4 clear_color;
     };
 
     Renderer(const options&);

src/render/UI.cpp

@@ -1,8 +1,9 @@
 #include "UI.hpp"
 
-#include "imgui_impl_glfw.h"
-#include "imgui_impl_opengl3.h"
-#include "pass/Program.hpp"
+#include <imgui.h>
+#include <imgui_impl_glfw.h>
+#include <imgui_impl_opengl3.h>
+#include "../state.h"
 #include "../contouring/Abstract.hpp"
 #include "../world/materials.hpp"
 
@@ -88,7 +89,7 @@ UI::Actions UI::draw(options &options, state &state, const reports &reports, GLu
                 actions = actions | Actions::RendererSharders;
             }
         }
-        if (ImGui::ColorEdit3("Fog color", (float *)&options.renderer.clear_color)) {
+        if (ImGui::ColorEdit3("Fog color", &options.renderer.clear_color[0])) {
             actions = actions | Actions::ClearColor;
         }
         if (ImGui::Checkbox("Wireframe", &options.renderer.wireframe))
@@ -105,7 +106,9 @@ UI::Actions UI::draw(options &options, state &state, const reports &reports, GLu
         ImGui::PlotHistogram("Count", reports.world.chunk_count.buffer.get(), reports.world.chunk_count.size, 0, std::to_string(reports.world.chunk_count.current()).c_str(), 0);
         ImGui::PlotHistogram("Loading", reports.world.chunk_load.buffer.get(), reports.world.chunk_load.size, 0, std::to_string(reports.world.chunk_load.current()).c_str(), 0);
         ImGui::PlotHistogram("Saving", reports.world.chunk_unload.buffer.get(), reports.world.chunk_unload.size, 0, std::to_string(reports.world.chunk_unload.current()).c_str(), 0);
+        ImGui::PlotHistogram("Regions", reports.world.region_count.buffer.get(), reports.world.region_count.size, 0, std::to_string(reports.world.region_count.current()).c_str(), 0);
         ImGui::Separator();
+        ImGui::Text("Path: %s", options.world.folderPath.c_str());
         if (ImGui::SliderInt("Load distance", &options.world.loadDistance, 1, options.world.keepDistance) |
             ImGui::SliderInt("Keep distance", &options.world.keepDistance, options.world.loadDistance + 1, 21)) {
             actions = actions | Actions::World;

src/render/UI.hpp

@@ -1,10 +1,11 @@
 #pragma once
 
-#include "imgui.h"
 #include <GL/glew.h>
 #include <GLFW/glfw3.h>
-#include "../state.h"
 
+class options;
+class state;
+class reports;
 namespace UI {
     /// Retro actions to state
     enum class Actions {

src/render/buffer/VertexData.hpp

@@ -7,6 +7,8 @@
 namespace buffer {
     /// Vertex properties
     struct VertexData {
+        VertexData(const glm::vec3 &position, GLushort material, const glm::vec3 &normal):
+            Position(position), Material(material), Normal(normal) { }
         glm::vec3 Position;
         GLushort Material;
         glm::vec3 Normal;

src/render/pass/MainProgram.cpp

@@ -8,13 +8,13 @@ MainProgram::MainProgram(const MainProgram::options& opts): Program() {
 
     std::vector<std::string> flags;
     if(opts.pbr)
-        flags.push_back("PBR");
+        flags.emplace_back("PBR");
     if(opts.triplanar)
-        flags.push_back("TRIPLANAR");
+        flags.emplace_back("TRIPLANAR");
     if (opts.fog)
-        flags.push_back("FOG");
+        flags.emplace_back("FOG");
     if (opts.blend)
-        flags.push_back("BLEND");
+        flags.emplace_back("BLEND");
 
     std::vector<Shader*> shaders;
     shaders.push_back(loadShader(GL_VERTEX_SHADER, flags));

src/render/pass/Shader.cpp

@@ -1,6 +1,5 @@
 #include "Shader.hpp"
 
-#include <cassert>
 #include <iostream>
 #include <fstream>
 #include <sstream>

src/state.h

@@ -16,14 +16,14 @@
 #include "control/Camera.hpp"
 #include "contouring/index.hpp"
 
-inline ImColor fromHex(const std::string& str) {
+inline glm::vec4 fromHex(const std::string& str) {
     int rgb[3] = {UCHAR_MAX};
     sscanf(str.c_str() + 1, "%02X%02X%02X", (unsigned int *)&rgb[0], (unsigned int *)&rgb[1], (unsigned int *)&rgb[2]);
-    return ImColor(rgb[0], rgb[1], rgb[2]);
+    return glm::vec4(rgb[0] * 1.f / UCHAR_MAX, rgb[1] * 1.f / UCHAR_MAX, rgb[2] * 1.f / UCHAR_MAX, 1);
 }
-inline std::string toHexa(const ImVec4& rgba) {
+inline std::string toHexa(const glm::vec4& rgb) {
     auto out = (char*)malloc(8 * sizeof(char));
-    sprintf(out, "#%02X%02X%02X", (int)(rgba.x * UCHAR_MAX), (int)(rgba.y * UCHAR_MAX), (int)(rgba.z * UCHAR_MAX));
+    sprintf(out, "#%02X%02X%02X", (int)(rgb.x * UCHAR_MAX), (int)(rgb.y * UCHAR_MAX), (int)(rgb.z * UCHAR_MAX));
     return std::string(out);
 }
 
@@ -49,6 +49,7 @@ struct options {
 
         world.loadDistance = config["world"]["load_distance"].value_or(world.loadDistance);
         world.keepDistance = config["world"]["keep_distance"].value_or(world.keepDistance);
+        world.folderPath = config["world"]["path"].value_or(world.folderPath);
         voxel_size = config["world"]["unit_size"].value_or(1.f);
 
         culling = config["mesh"]["culling"].value_or(true);
@@ -98,7 +99,8 @@ struct options {
         config.insert_or_assign("world", toml::table({
             {"load_distance", world.loadDistance},
             {"keep_distance", world.keepDistance},
-            {"unit_size", voxel_size}
+            {"unit_size", voxel_size},
+            {"path", world.folderPath}
         }));
         config.insert_or_assign("mesh", toml::table({
             {"culling", culling},

src/world/Chunk.cpp

@@ -1,12 +1,13 @@
 #include "Chunk.hpp"
 
-#include <FastNoiseSIMD.h>
 #include "materials.hpp"
+#include <algorithm>
 
 using namespace world;
 
 #define DENSITY 0.f
 #define GRANULARITY 30.f
+#define RLE 1
 
 Chunk::Chunk(const chunk_pos& pos, Generator& rnd) {
     const auto [densitySet, materialSet] = rnd.getChunk(pos, CHUNK_LENGTH);
@@ -19,8 +20,60 @@ Chunk::Chunk(const chunk_pos& pos, Generator& rnd) {
     FastNoiseSIMD::FreeNoiseSet(densitySet);
     FastNoiseSIMD::FreeNoiseSet(materialSet);
 }
+Chunk::Chunk(std::istream& str) {
+#ifdef RLE
+    ushort i = 0;
+    while(!str.eof()) {
+        ushort count;
+        Voxel voxel;
+        str.read(reinterpret_cast<char *>(&count), sizeof(count));
+        str.read(reinterpret_cast<char *>(&voxel.Density), sizeof(Voxel::Density));
+        str.read(reinterpret_cast<char *>(&voxel.Material), sizeof(Voxel::Material));
+        str.peek();
+        for (; count > 0; count--) {
+            voxels[i] = voxel;
+            i++;
+        }
+    }
+    assert(("Mismatch data length", i == CHUNK_SIZE-1));
+#else
+    for(auto& voxel: voxels) {
+        str.read(reinterpret_cast<char *>(&voxel.Density), sizeof(Voxel::Density));
+        str.read(reinterpret_cast<char *>(&voxel.Material), sizeof(Voxel::Material));
+    }
+#endif
+}
 Chunk::~Chunk() { }
 
+void Chunk::write(std::ostream& str) const {
+#ifdef RLE
+    auto it = voxels.begin();
+    ushort counter = 1;
+    Voxel current = *it;
+    while(true) {
+        it++;
+        const auto end = (it == voxels.end());
+        if(end || current.Density != it->Density || current.Material != it->Material) {
+            str.write(reinterpret_cast<char *>(&counter), sizeof(counter));
+            str.write(reinterpret_cast<char *>(&current.Density), sizeof(current.Density));
+            str.write(reinterpret_cast<char *>(&current.Material), sizeof(current.Material));
+            if(end)
+                break;
+
+            current = *it;
+            counter = 1;
+        } else {
+            counter++;
+        }
+    }
+#else
+    for(auto current: voxels) {
+        str.write(reinterpret_cast<char *>(&current.Density), sizeof(current.Density));
+        str.write(reinterpret_cast<char *>(&current.Material), sizeof(current.Material));
+    }
+#endif
+}
+
 std::optional<Faces> Chunk::update() {
     if(upToDate) {
         return {};

src/world/Chunk.hpp

@@ -3,9 +3,9 @@
 #include "Generator.hpp"
 #include "Voxel.hpp"
 #include "../data/geometry/Faces.hpp"
+#include <sstream>
 
 /// Chunk length
-#define CHUNK_LENGTH 32
 #define CHUNK_LENGTH2 (CHUNK_LENGTH * CHUNK_LENGTH)
 #define CHUNK_SIZE (CHUNK_LENGTH2 * CHUNK_LENGTH)
 
@@ -15,6 +15,7 @@ namespace world {
     struct Chunk {
     public:
         Chunk(const chunk_pos& pos, Generator& rnd);
+        Chunk(std::istream& str);
         ~Chunk();
 
         /// Update voxels
@@ -48,6 +49,11 @@ namespace world {
             set(idx, val);
             return Item{res.Density, res.Material};
         }
+        // Is player modified
+        inline bool isModified() const { return modified; }
+        // Write to file.
+        // Using RLE
+        void write(std::ostream& str) const;
 
         static inline chunk_voxel_pos getPosition(ushort idx) {
             return chunk_voxel_pos(idx / CHUNK_LENGTH2, (idx / CHUNK_LENGTH) % CHUNK_LENGTH, idx % CHUNK_LENGTH);

src/world/Generator.hpp

@@ -2,7 +2,7 @@
 
 #include <FastNoiseSIMD.h>
 #include <tuple>
-#include "../data/glm.hpp"
+#include "position.h"
 
 namespace world {
     /// Noise generator

src/world/Universe.cpp

@@ -1,86 +1,152 @@
 #include "Universe.hpp"
 
-#include "../contouring/Dummy.hpp"
 #include <Remotery.h>
+#include <filesystem>
+
+#include "../contouring/Dummy.hpp"
+#include "Chunk.hpp"
 
 using namespace world;
 
-Universe::Universe(const Universe::options &options): loadPool(2), contouring(std::make_shared<contouring::Dummy>()) {
+Universe::Universe(const Universe::options &options): generator(42), regionDict("content/zstd.dict"), contouring(std::make_shared<contouring::Dummy>()) {
     setOptions(options);
+    folderPath = options.folderPath;
+    struct vec_istream: std::streambuf {
+        vec_istream(std::vector<char> &vec) {
+            this->setg(&vec[0], &vec[0], &vec[0] + vec.size());
+        }
+    };
+    running = true;
+    std::filesystem::create_directories(folderPath);
+
+    // Load workers
+    for (size_t i = 0; i < 4; i++) {
+        loadWorkers.emplace_back([&] {
+            const auto ctx = regionDict.make_reader();
+            while (running) {
+                chunk_pos pos;
+                loadQueue.wait();
+                if (loadQueue.pop(pos)) {
+                    //MAYBE: loadQueue.take to avoid duplicated work on fast move
+                    rmt_ScopedCPUSample(ProcessLoad, 0);
+                    const region_pos rPos = glm::divide(pos, region_chunk_pos(REGION_LENGTH));
+                    const region_chunk_pos cPos = glm::modulo(pos, region_chunk_pos(REGION_LENGTH));
+                    const auto reg = getRegion(rPos);
+                    Region::data data;
+                    if(reg->read(cPos, ctx, data)) {
+                        rmt_ScopedCPUSample(ProcessRead, 0);
+                        vec_istream idata(data);
+                        std::istream iss(&idata);
+                        loadedQueue.push({pos, std::make_shared<Chunk>(iss)});
+                    } else {
+                        rmt_ScopedCPUSample(ProcessGenerate, 0);
+                        loadedQueue.push({pos, std::make_shared<Chunk>(pos, generator)});
+                    }
+                }
+            }
+        });
+    }
+
+    // Save workers
+    for (size_t i = 0; i < 2; i++) {
+        saveWorkers.emplace_back([&] {
+            const auto ctx = regionDict.make_writer();
+            while (running) {
+                robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> task;
+                saveQueue.wait();
+                if (saveQueue.pop(task) && task.second->isModified()) {
+                    //MAYBE: queue.take to avoid concurent write or duplicated work on fast move
+                    rmt_ScopedCPUSample(ProcessSave, 0);
+                    std::ostringstream out;
+                    task.second->write(out);
+                    const region_pos rPos = glm::divide(task.first, region_chunk_pos(REGION_LENGTH));
+                    const region_chunk_pos cPos = glm::modulo(task.first, region_chunk_pos(REGION_LENGTH));
+                    const auto reg = getRegion(rPos);
+                    reg->write(cPos, ctx, out.str());
+                }
+            }
+        });
+    }
 }
 Universe::~Universe() {
     contouring = NULL;
-}
 
-Universe::LoadPool::LoadPool(size_t count) {
-    for (size_t i = 0; i < count; i++) {
-        workers.push_back(std::thread([&] {
-            while (running) {
-                chunk_pos ctx;
-                loadQueue.wait();
-                if (loadQueue.pop(ctx)) {
-                    rmt_ScopedCPUSample(ProcessGenerate, 0);
-                    loadedQueue.push({ctx, std::make_shared<Chunk>(ctx, generator)});
-                }
-            }
-        }));
+    // Save all
+    for(auto& pair: chunks) {
+        saveQueue.push(pair);
     }
-}
-Universe::LoadPool::~LoadPool() {
+    if (auto size = saveQueue.size(); size > 0) {
+        std::cout << std::endl;
+        do {
+            std::cout << "\rSaving... " << size << "            " << std::flush;
+            std::this_thread::sleep_for(std::chrono::microseconds(500));
+            size = saveQueue.size();
+        } while (size > 0);
+        std::cout << std::endl;
+    }
+
     running = false;
     loadQueue.notify();
+    saveQueue.notify();
 
-    for (auto &worker : workers) {
+    for (auto &worker: loadWorkers) {
+        if (worker.joinable())
+            worker.join();
+    }
+    for (auto &worker: saveWorkers) {
         if (worker.joinable())
             worker.join();
     }
 }
-inline void Universe::LoadPool::push(const chunk_pos &pos, int weight) { loadQueue.push(pos, weight); }
-inline bool Universe::LoadPool::pop(robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> &out) { return loadedQueue.pop(out); }
-inline size_t Universe::LoadPool::size() { return loadQueue.size(); }
+
+std::shared_ptr<Region> Universe::getRegion(const region_pos& pos) {
+    std::shared_lock lock(regionMutex);
+    const auto it = regionCache.find(pos);
+    if(it == regionCache.end()) {
+        lock.unlock();
+        const auto reg = std::make_shared<Region>(folderPath, pos);
+        std::unique_lock u_lock(regionMutex);
+        return regionCache.insert({pos, reg}).first->second;
+    } else {
+        return it->second;
+    }
+}
 
 void Universe::update(const camera_pos& pos, Universe::report& rep) {
     const chunk_pos newPos = glm::divide(pos, chunk_voxel_pos(CHUNK_LENGTH));
-    const auto chunkChange = newPos != last_pos;
+    const auto chunkChange = last_pos != newPos;
     last_pos = newPos;
     rmt_ScopedCPUSample(Universe, 0);
 
     // Update alive chunks
     {
         rmt_ScopedCPUSample(Update, 0);
-        for (auto &[chunkPos, chunk]: chunks) {
-            if (glm::length2(last_pos - chunkPos) > keepDistance * keepDistance
-                    && unloadQueue.push(chunkPos)) {
-                //TODO: unloadCount++;
-                continue;
-            }
-
-            if (const auto neighbors = chunk->update()) {
-                contouring->onUpdate(chunkPos, chunks, neighbors.value()); //TODO: get update update_type(simple(pos), complex)
-            } else if (chunkChange) { //NOTE: must be solved before octrees
-                contouring->onNotify(chunkPos, chunks);
+        auto it = chunks.begin();
+        while (it != chunks.end()) {
+            if (glm::length2(last_pos - it->first) > keepDistance * keepDistance) {
+                saveQueue.push(*it);
+                it = chunks.erase(it);
+            } else {
+                if (const auto neighbors = it->second->update()) {
+                    contouring->onUpdate(it->first, chunks, neighbors.value()); //TODO: get update update_type(simple(pos), complex)
+                } else if (chunkChange) {
+                    contouring->onNotify(it->first, chunks);
+                }
+                ++it;
             }
         }
     }
+    rep.chunk_unload.push(saveQueue.size());
     {
         rmt_ScopedCPUSample(Contouring, 0);
         contouring->update(pos);
-    }
-
-    rep.chunk_unload.push(unloadQueue.size());
-    // Unload dead chunks
-    {
-        rmt_ScopedCPUSample(Unload, 0);
-        for (size_t i = 0; i < 256 && !unloadQueue.empty(); i++) {
-            chunks.erase(unloadQueue.pop());
-            //TODO: save to file
-        }
+        //MAYBE: if(chunkChange) contouring->notify(chunks);
     }
 
     // Find missing chunks
     if(chunkChange) {
         rmt_ScopedCPUSample(ToLoad, 0);
-        //TODO: circle point algo
+        //NOTE: need dist so no easy sphere fill
         for (int x = -loadDistance; x <= loadDistance; x++) {
         for (int y = -loadDistance; y <= loadDistance; y++) {
         for (int z = -loadDistance; z <= loadDistance; z++) {
@@ -88,23 +154,38 @@ void Universe::update(const camera_pos& pos, Universe::report& rep) {
             if (dist2 <= loadDistance * loadDistance) {
                 const chunk_pos p = last_pos + glm::ivec3(x, y, z);
                 if (chunks.find(p) == chunks.end()) {
-                    loadPool.push(p, -dist2);
+                    loadQueue.push(p, -dist2);
                 }
             }
         }}}
     }
 
-    rep.chunk_load.push(loadPool.size());
+    rep.chunk_load.push(loadQueue.size());
     // Loaded chunks
     {
         rmt_ScopedCPUSample(Load, 0);
         robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> loaded;
-        while (loadPool.pop(loaded)) {
+        while (loadedQueue.pop(loaded)) {
             chunks.insert(loaded);
             contouring->onUpdate(loaded.first, chunks, Faces::All);
         }
     }
     rep.chunk_count.push(chunks.size());
+
+    {
+        rmt_ScopedCPUSample(Region, 0);
+        std::unique_lock lock(regionMutex);
+        const auto me = glm::divide(last_pos, glm::ivec3(REGION_LENGTH));
+        for (auto it = regionCache.begin(); it != regionCache.end();) {
+            if (glm::length2(it->first - me) > keepDistance) {
+                std::cout << "rem" << std::endl;
+                it = regionCache.erase(it);
+            } else {
+                it++;
+            }
+        }
+    }
+    rep.region_count.push(regionCache.size());
 }
 void Universe::setOptions(const Universe::options& options) {
     loadDistance = options.loadDistance;

src/world/Universe.hpp

@@ -1,13 +1,19 @@
 #pragma once
 
+#include <string>
 #include <memory>
 #include <thread>
-#include "../data/unique_queue.hpp"
+#include <shared_mutex>
+#include "../data/math.hpp"
 #include "../data/safe_queue.hpp"
 #include "../data/safe_priority_queue.hpp"
 #include "../data/circular_buffer.hpp"
-#include "Chunk.hpp"
 #include "../data/geometry/Ray.hpp"
+#include "forward.h"
+#include "Voxel.hpp"
+#include "region/index.hpp"
+#include "Generator.hpp"
+typedef glm::vec3 camera_pos;
 
 #define REPORT_BUFFER_SIZE 128
 
@@ -27,6 +33,8 @@ namespace world {
             int loadDistance = 5;
             /// Radius in chunks to keep in memory
             int keepDistance = 6;
+            /// Storage path
+            std::string folderPath = "world";
         };
         /// Reports to UI
         struct report {
@@ -36,9 +44,11 @@ namespace world {
             circular_buffer<float> chunk_load = circular_buffer<float>(REPORT_BUFFER_SIZE, 0);
             /// Saved chunks
             circular_buffer<float> chunk_unload = circular_buffer<float>(REPORT_BUFFER_SIZE, 0);
+            /// Regions in memory
+            circular_buffer<float> region_count = circular_buffer<float>(REPORT_BUFFER_SIZE, 0);
         };
 
-        Universe(const options&);
+        Universe(const options &);
         ~Universe();
 
         /// Update physics and contouring
@@ -65,7 +75,7 @@ namespace world {
         chunk_pos last_pos = chunk_pos(INT_MAX);
 
         /// Data
-        robin_hood::unordered_map<chunk_pos, std::shared_ptr<Chunk>> chunks;
+        chunk_map chunks;
 
         std::optional<std::shared_ptr<Chunk>> at(const chunk_pos& pos) const {
             const auto it = chunks.find(pos);
@@ -74,29 +84,24 @@ namespace world {
             return {it->second};
         }
 
-        /// Generating worker pool
-        class LoadPool {
-        public:
-            LoadPool(size_t size);
-            ~LoadPool();
+        Generator generator;
 
-            Generator generator;
-            inline void push(const chunk_pos &pos, int weight);
-            inline bool pop(robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> &out);
-            inline size_t size();
+        bool running = true;
+        std::vector<std::thread> loadWorkers;
+        safe_priority_queue<chunk_pos, int> loadQueue;
+        safe_queue<robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>>> loadedQueue;
 
-        private:
-            std::vector<std::thread> workers;
-            bool running = true;
-
-            safe_priority_queue<chunk_pos, int> loadQueue;
-            safe_queue<robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>>> loadedQueue;
-        };
-        LoadPool loadPool;
-        unique_queue<chunk_pos> unloadQueue;
+        std::vector<std::thread> saveWorkers;
+        safe_queue<robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>>> saveQueue; //NOTE: consider const Chunk
 
         int loadDistance;
         int keepDistance;
+        std::string folderPath;
+
+        std::shared_mutex regionMutex; //MAYBE: shared_guard
+        robin_hood::unordered_map<region_pos, std::shared_ptr<Region>> regionCache;
+        dict_set regionDict;
+        std::shared_ptr<Region> getRegion(const region_pos &);
 
         /// Contouring worker
         std::shared_ptr<contouring::Abstract> contouring;

src/world/Voxel.hpp

@@ -6,6 +6,8 @@ namespace world {
     /// Universe unit
     struct Voxel {
         /// Quantity of material
+        /// FIXME: low density area are cheatty
+        /// @note v < iso * UCHAR_MAX are useless
         unsigned char Density;
         /// Material type
         /// @see world::materials

src/world/forward.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <robin_hood.h>
+#include "position.h"
+
+namespace world {
+    class Chunk;
+    typedef robin_hood::unordered_map<chunk_pos, std::shared_ptr<Chunk>> chunk_map;
+}

src/world/position.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "../data/glm.hpp"
+
+#define CHUNK_LENGTH 32
+#define REGION_LENGTH 32
+
+typedef glm::lvec3 voxel_pos;
+typedef glm::ivec3 chunk_pos;
+typedef glm::ucvec3 chunk_voxel_pos;
+typedef glm::ivec3 region_pos;
+typedef glm::ucvec3 region_chunk_pos;

src/world/region/File.cpp

@@ -0,0 +1,178 @@
+#include "File.hpp"
+
+#include <filesystem>
+
+using namespace world;
+
+#define REMOVE_CORRUPTED 1
+
+FileRegion::FileRegion(const std::string &folderPath, const region_pos &pos) {
+    path = folderPath + '/' + std::to_string(pos.x) + '.' +
+        std::to_string(pos.y) + '.' + std::to_string(pos.z) + ".map";
+
+    load();
+}
+FileRegion::~FileRegion() {
+    file.close();
+}
+
+void FileRegion::load() {
+    std::unique_lock lock(mutex);
+
+    file.open(path, std::ios::in | std::ios::binary);
+    if(!file.good()) {
+        return;
+    }
+    // Read header
+    ushort chunkCount; //NOTE: pretty useless
+    file.read(reinterpret_cast<char *>(&chunkCount), sizeof(chunkCount));
+
+    while (!file.eof()) {
+        // Read pos
+        region_chunk_pos pos;
+        file.read(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
+        file.read(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
+        file.read(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
+
+        //NOTE: align uchar pos
+        if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
+            file.ignore(1);
+        }
+
+        // Read size
+        ushort size = 0;
+        file.read(reinterpret_cast<char *>(&size), sizeof(size));
+
+        // Ignore content
+        if(!index.insert({pos, std::make_pair(size, file.tellg())}).second) {
+            std::cout << "Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << std::endl;
+        }
+        file.ignore(size);
+        file.peek();
+    }
+
+    if(file.bad()) {
+        std::cout << "region corrupted read " << path << std::endl;
+    }
+
+    assert(index.size() == chunkCount);
+}
+bool FileRegion::read(const region_chunk_pos& pos, const read_ctx& ctx, data& out) {
+    std::unique_lock lock(mutex);
+
+    const auto it = index.find(pos);
+    if (it == index.end())
+        return false;
+
+    auto in = std::make_unique<data>();
+    in->resize(it->second.first);
+    file.seekg(it->second.second);
+    file.read(in->data(), in->size());
+
+    const auto maxSize = ZSTD_getFrameContentSize(in->data(), in->size());
+    out.resize(maxSize);
+    const auto actualSize = ZSTD_decompress_usingDDict(ctx.ctx, out.data(), out.size(), in->data(), in->size(), ctx.dict);
+    if(ZSTD_isError(actualSize)) {
+        std::cout << "Corrupted region chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
+            << ZSTD_getErrorName(actualSize) << std::endl;
+#ifdef REMOVE_CORRUPTED
+            std::cout << "Removing" << std::endl;
+            index.erase(it);
+            lock.unlock();
+            save(std::nullopt);
+#endif
+        return false;
+    }
+    out.resize(actualSize);
+    return true;
+}
+void FileRegion::write(const region_chunk_pos& pos, const write_ctx& ctx, const std::string_view& in) {
+    const auto maxSize = ZSTD_compressBound(in.size());
+    auto buffer = std::make_unique<FileRegion::data>();
+    buffer->resize(maxSize);
+
+    const auto actualSize = ZSTD_compress_usingCDict(ctx.ctx, buffer->data(), buffer->capacity(), in.data(), in.size(), ctx.dict);
+    if (ZSTD_isError(actualSize)) {
+        std::cout << "Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
+            << ZSTD_getErrorName(actualSize) << std::endl;
+        return;
+    }
+    buffer->resize(actualSize);
+    save({{pos, std::move(buffer)}});
+}
+
+void FileRegion::save(std::optional<std::pair<region_chunk_pos, std::unique_ptr<FileRegion::data>>> added) {
+    std::unique_lock lock(mutex);
+
+    const auto tmpPath = path + ".tmp";
+
+    std::ofstream tmpFile(tmpPath, std::ios::out | std::ios::binary);
+    if (!tmpFile.good()) {
+        std::cout << "Corrupted region path: " << tmpPath << std::endl;
+        return;
+    }
+
+    { // Write header
+        ushort size = index.size() + (added.has_value() ? 1 : 0);
+        tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
+    }
+
+    data tmp;
+    for(const auto& chunk: index) {
+        { // Write pos
+            region_chunk_pos pos = chunk.first;
+            tmpFile.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
+            tmpFile.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
+            tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
+        }
+
+        //NOTE: align uchar pos
+        if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
+            tmpFile.put(0);
+            //MAYBE: store usefull uchar flags
+        }
+
+        // Write size
+        auto size = chunk.second.first;
+        tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
+
+        // Write content
+        tmp.resize(size);
+        file.seekg(chunk.second.second);
+        file.read(tmp.data(), size);
+        tmpFile.write(tmp.data(), size);
+    }
+    if(added.has_value()) {
+        { // Write pos
+            region_chunk_pos pos = added.value().first;
+            tmpFile.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
+            tmpFile.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
+            tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
+        }
+
+        //NOTE: align uchar pos
+        if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
+            tmpFile.put(0);
+            //MAYBE: store usefull uchar flags
+        }
+
+        // Write size
+        auto size = added.value().second->size();
+        tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
+
+        // Write content
+        tmpFile.write(added.value().second->data(), size);
+    }
+
+    if (!tmpFile.good()) {
+        std::cout << "region corrupted write " << tmpPath << std::endl;
+        tmpFile.close();
+        return;
+    }
+    tmpFile.close();
+
+    index.clear();
+    file.close();
+    std::filesystem::rename(tmpPath, path);
+    load();
+}

src/world/region/File.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <shared_mutex>
+#include "../forward.h"
+#include "common.hpp"
+#include "../../data/math.hpp"
+
+namespace world {
+    ///Group of chunks saved as a single file only pointer
+    class FileRegion {
+    public:
+        FileRegion(const std::string& folderPath, const region_pos &pos);
+        ~FileRegion();
+
+        typedef std::vector<char> data;
+
+        bool read(const region_chunk_pos &pos, const read_ctx& ctx, data &out);
+        void write(const region_chunk_pos &pos, const write_ctx& ctx, const std::string_view &in);
+
+    private:
+        void save(std::optional<std::pair<region_chunk_pos, std::unique_ptr<FileRegion::data>>> added);
+
+        std::string path;
+        //TODO: use tickets to remove unused regions
+
+        std::shared_mutex mutex;
+        std::ifstream file;
+        robin_hood::unordered_flat_map<region_chunk_pos, std::pair<ushort, std::streampos>> index;
+
+        void load();
+    };
+}

src/world/region/Memory.cpp

@@ -0,0 +1,178 @@
+#include "Memory.hpp"
+
+using namespace world;
+
+#define REMOVE_CORRUPTED 1
+#define LAZYNESS 8
+
+MemoryRegion::MemoryRegion(const std::string &folderPath, const region_pos &pos) {
+    path = folderPath + '/' + std::to_string(pos.x) + '.' +
+        std::to_string(pos.y) + '.' + std::to_string(pos.z) + ".map";
+
+    load();
+}
+MemoryRegion::~MemoryRegion() {
+    if(!content.empty())
+        save(changed);
+
+    std::unique_lock lock(mutex);
+    auto it = content.begin();
+    while(it != content.end()) {
+        delete it->second;
+        it = content.erase(it);
+    }
+}
+
+void MemoryRegion::load() {
+    std::unique_lock lock(mutex);
+
+    std::ifstream file;
+    file.open(path, std::ios::in | std::ios::binary);
+    if(!file.good()) {
+        return;
+    }
+
+    // Read header
+    ushort chunkCount; //NOTE: pretty useless
+    file.read(reinterpret_cast<char *>(&chunkCount), sizeof(chunkCount));
+
+    while (!file.eof()) {
+        // Read pos
+        region_chunk_pos pos;
+        file.read(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
+        file.read(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
+        file.read(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
+
+        //NOTE: align uchar pos
+        if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
+            file.ignore(1);
+        }
+
+        // Read size
+        ushort size = 0;
+        file.read(reinterpret_cast<char *>(&size), sizeof(size));
+
+        // Read content
+        const auto data = new MemoryRegion::data();
+        data->resize(size);
+        file.read(data->data(), data->size());
+        if(!content.insert({pos, data}).second) {
+            std::cout << "Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << std::endl;
+        }
+        file.peek();
+    }
+
+    if(file.bad()) {
+        std::cout << "region corrupted read " << path << std::endl;
+    }
+    assert(content.size() == chunkCount);
+    file.close();
+}
+bool MemoryRegion::read(const region_chunk_pos& pos, const read_ctx& ctx, data& out) {
+    std::shared_lock lock(mutex);
+
+    const auto it = content.find(pos);
+    if (it == content.end())
+        return false;
+
+    auto &in = it->second;
+
+    const auto maxSize = ZSTD_getFrameContentSize(in->data(), in->size());
+    out.resize(maxSize);
+    const auto actualSize = ZSTD_decompress_usingDDict(ctx.ctx, out.data(), out.size(), in->data(), in->size(), ctx.dict);
+    if(ZSTD_isError(actualSize)) {
+        std::cout << "Corrupted region chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
+            << ZSTD_getErrorName(actualSize) << std::endl;
+#ifdef REMOVE_CORRUPTED
+            std::cout << "Removing" << std::endl;
+            lock.unlock();
+            {
+                std::unique_lock ulock(mutex);
+                content.erase(it);
+            }
+            save(true);
+#endif
+        return false;
+    }
+    out.resize(actualSize);
+    return true;
+}
+void MemoryRegion::write(const region_chunk_pos& pos, const write_ctx& ctx, const std::string_view& in) {
+    const auto maxSize = ZSTD_compressBound(in.size());
+    const auto buffer = new MemoryRegion::data();
+    buffer->resize(maxSize);
+
+    const auto actualSize = ZSTD_compress_usingCDict(ctx.ctx, buffer->data(), buffer->capacity(), in.data(), in.size(), ctx.dict);
+    if (ZSTD_isError(actualSize)) {
+        std::cout << "Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
+            << ZSTD_getErrorName(actualSize) << std::endl;
+        return;
+    }
+    buffer->resize(actualSize);
+
+    {
+        std::unique_lock lock(mutex);
+
+        // Save buffer
+        const auto it = content.find(pos);
+        if (it != content.end())
+        {
+            delete it->second;
+            content.erase(it);
+        }
+        content.insert({pos, buffer});
+        changed = true;
+    }
+    save(false);
+}
+
+void MemoryRegion::save(bool force) {
+    if(!force && rand() % LAZYNESS == 0)
+        return;
+
+    std::unique_lock lock(mutex);
+
+    std::ofstream file(path, std::ios::out | std::ios::binary);
+    if (!file.good()) {
+        std::cout << "Corrupted region path: " << path << std::endl;
+        return;
+    }
+
+    { // Write header
+        ushort size = (ushort)content.size();
+        file.write(reinterpret_cast<char *>(&size), sizeof(size));
+    }
+
+    for(const auto& chunk: content) {
+        assert(chunk.second->size() < USHRT_MAX);
+        auto size = (ushort)chunk.second->size();
+        const auto out = chunk.second->data();
+
+        { // Write pos
+            region_chunk_pos pos = chunk.first;
+            file.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
+            file.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
+            file.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
+        }
+
+        //NOTE: align uchar pos
+        if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
+            file.put(0);
+            //MAYBE: store usefull uchar flags
+        }
+
+        // Write size
+        file.write(reinterpret_cast<char *>(&size), sizeof(size));
+
+        // Write content
+        file.write(out, size);
+    }
+
+    if (!file.good()) {
+        std::cout << "region corrupted write " << path << std::endl;
+        file.close();
+        return;
+    }
+    file.close();
+    changed = false;
+}

src/world/region/Memory.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <shared_mutex>
+#include <zstd.h>
+#include "../forward.h"
+#include "common.hpp"
+#include "../../data/math.hpp"
+
+namespace world {
+    ///Group of chunks saved as a single file in memory
+    class MemoryRegion {
+    public:
+        MemoryRegion(const std::string& folderPath, const region_pos &pos);
+        ~MemoryRegion();
+
+        typedef std::vector<char> data;
+
+        bool read(const region_chunk_pos &pos, const read_ctx& ctx, data &out);
+        void write(const region_chunk_pos &pos, const write_ctx& ctx, const std::string_view &in);
+
+    private:
+        void save(bool force = true);
+
+        std::string path;
+        //TODO: use tickets to remove unused regions
+
+        std::shared_mutex mutex;
+        robin_hood::unordered_flat_map<region_chunk_pos, data*> content;
+        bool changed = false;
+
+        void load();
+    };
+}

src/world/region/common.hpp

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <zstd.h>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <cassert>
+
+namespace world {
+    struct read_ctx {
+        ~read_ctx() {
+            ZSTD_freeDCtx(ctx);
+        }
+        ZSTD_DCtx *ctx;
+        ZSTD_DDict *dict;
+    };
+    struct write_ctx {
+        ~write_ctx() {
+            ZSTD_freeCCtx(ctx);
+        }
+        ZSTD_CCtx *ctx;
+        ZSTD_CDict *dict;
+    };
+
+    class dict_set {
+    public:
+        dict_set(const std::string& path) {
+            std::ifstream is(path, std::ios::in | std::ios::binary | std::ios::ate);
+            if(!is.good()) {
+                std::cout << "missing dict " << path << std::endl;
+                exit(1);
+            }
+            const auto end = is.tellg();
+            is.seekg(0, std::ios::beg);
+            std::vector<char> dict(end - is.tellg());
+            is.read(dict.data(), dict.size());
+            is.close();
+            c = ZSTD_createCDict(dict.data(), dict.size(), ZSTD_CLEVEL_DEFAULT);
+            assert(c != NULL);
+            d = ZSTD_createDDict(dict.data(), dict.size());
+            assert(d != NULL);
+        }
+        ~dict_set() {
+            ZSTD_freeCDict(c);
+            ZSTD_freeDDict(d);
+        }
+
+        read_ctx make_reader() const {
+            return read_ctx{ZSTD_createDCtx(), d};
+        }
+        write_ctx make_writer() const {
+            return write_ctx{ZSTD_createCCtx(), c};
+        }
+    private:
+        ZSTD_CDict *c;
+        ZSTD_DDict *d;
+    };
+}

src/world/region/index.hpp

@@ -0,0 +1,7 @@
+#ifdef LOW_MEMORY
+#include "File.hpp"
+namespace world {typedef FileRegion Region;}
+#else
+#include "Memory.hpp"
+namespace world {typedef MemoryRegion Region;}
+#endif