From 2bb02bef1f0981dfd2c93d333462cfee29efc5d2 Mon Sep 17 00:00:00 2001 From: Shu Date: Fri, 6 Nov 2020 17:32:00 +0100 Subject: [PATCH] Regions level preload (1:32) --- src/client/config.hpp | 4 + src/client/world/Area.hpp | 2 + src/client/world/DistantUniverse.cpp | 123 +++++++++++++++++++++------ src/client/world/DistantUniverse.hpp | 3 +- src/client/world/Universe.hpp | 8 ++ src/core/net/data.hpp | 12 ++- src/core/net/io.hpp | 6 +- src/core/world/EdittableChunk.cpp | 1 + src/core/world/EdittableChunk.hpp | 13 +++ src/core/world/Voxel.hpp | 21 +++-- src/server/world/Chunk.cpp | 23 ++++- src/server/world/Chunk.hpp | 5 +- src/server/world/Universe.cpp | 63 +++++++++++--- src/server/world/region/File.cpp | 107 ++++++++++++++--------- src/server/world/region/File.hpp | 43 ++++++++-- src/server/world/region/Memory.cpp | 97 ++++++++++++--------- src/server/world/region/Memory.hpp | 35 ++++++-- 17 files changed, 412 insertions(+), 154 deletions(-) diff --git a/src/client/config.hpp b/src/client/config.hpp index 0e8416e..c987542 100644 --- a/src/client/config.hpp +++ b/src/client/config.hpp @@ -64,6 +64,8 @@ public: world.loadDistance = config["world"]["load_distance"].value_or(world.loadDistance); world.keepDistance = config["world"]["keep_distance"].value_or(world.keepDistance); + world.useAverages = config["world"]["use_averages"].value_or(world.useAverages); + world.trustMajorant = config["world"]["trust_majorant"].value_or(world.trustMajorant); voxel_density = config["world"]["voxel_density"].value_or(voxel_density); #ifndef STANDALONE @@ -140,6 +142,8 @@ public: config.insert_or_assign("world", toml::table({ {"load_distance", world.loadDistance}, {"keep_distance", world.keepDistance}, + {"use_averages", world.useAverages}, + {"trust_majorant", world.trustMajorant}, {"voxel_density", voxel_density} })); if(connection.has_value()) { diff --git a/src/client/world/Area.hpp b/src/client/world/Area.hpp index 6d464cd..3a574d8 100644 --- a/src/client/world/Area.hpp +++ b/src/client/world/Area.hpp @@ -16,6 +16,8 @@ namespace world::client { curvature = p.curvature; } + robin_hood::unordered_map> regionCache; + private: std::optional curvature; }; diff --git a/src/client/world/DistantUniverse.cpp b/src/client/world/DistantUniverse.cpp index 6902d38..36dda93 100644 --- a/src/client/world/DistantUniverse.cpp +++ b/src/client/world/DistantUniverse.cpp @@ -11,8 +11,8 @@ using namespace world::client; -DistantUniverse::DistantUniverse(const connection& ct, const options& opt, const std::string& contouring): Universe(contouring), - loadDistance(opt.loadDistance), keepDistance(opt.keepDistance), serverDistance(0), +DistantUniverse::DistantUniverse(const connection& ct, const Universe::options& opt, const std::string& contouring): + Universe(contouring), options(opt), serverDistance(0), peer(ct, [&](const data::out_view& buf, net::PacketFlags flags){ return onPacket(buf, flags); }) { } DistantUniverse::~DistantUniverse() { } @@ -33,11 +33,11 @@ void DistantUniverse::update(voxel_pos pos, float deltaTime) { const bool chunkChangeArea = (false && area.second->move(glm::vec3(deltaTime))) || chunkChange; // TODO: area.velocity const chunk_pos diff = glm::divide(pos - area.second->getOffset().as_voxel()); auto &chunks = area.second->setChunks(); - if (glm::length2(diff) <= glm::pow2(keepDistance + area.second->getChunks().getRadius())) { + if (glm::length2(diff) <= glm::pow2(options.keepDistance + area.second->getChunks().getRadius())) { ZoneScopedN("Alive"); auto it_c = chunks.begin(); while (it_c != chunks.end()) { - if (glm::length2(diff - it_c->first) > glm::pow2(keepDistance)) { + if (glm::length2(diff - it_c->first) > glm::pow2(options.keepDistance)) { it_c = chunks.erase(it_c); } else { if(const auto neighbors = std::dynamic_pointer_cast(it_c->second)->update(deltaTime, true /*MAYBE: random update*/)) { @@ -46,45 +46,77 @@ void DistantUniverse::update(voxel_pos pos, float deltaTime) { ++it_c; } } + //FIXME: Produce important duplicated (chunkChangeArea) + //MAYBE: if dist to unloaded chunk < X if (chunkChangeArea || mayQueryChunks) { // Request missing chunks ZoneScopedN("Missing"); - std::vector missing; + std::vector missingChunks; + std::vector missingRegions; std::vector missingDist; + const auto cl_area = std::dynamic_pointer_cast(area.second); //TODO: use easy sphere fill - const int queryDistance = std::min(loadDistance, serverDistance); + const int queryDistance = std::min(options.loadDistance, serverDistance); for (int x = -queryDistance; x <= queryDistance; x++) { for (int y = -queryDistance; y <= queryDistance; y++) { for (int z = -queryDistance; z <= queryDistance; z++) { const auto dist2 = x * x + y * y + z * z; const auto p = diff + chunk_pos(x, y, z); if (dist2 <= queryDistance * queryDistance && chunks.inRange(p)) { - if (chunks.find(p) != chunks.end()) { + if (auto it = chunks.find(p); it != chunks.end() && + std::dynamic_pointer_cast(it->second)->isTrusted(options.trustMajorant)) + //TODO: config accept perfect average + { contouring->onNotify(std::make_pair(area.first, p), diff, chunks); - } else { - if (missingDist.size() >= net::MAX_PENDING_CHUNK_COUNT) { - if (dist2 > missingDist.front()) - continue; - missingDist.erase(missingDist.begin()); - missing.erase(missing.begin()); - } - - auto it = missingDist.begin(); - auto itv = missing.begin(); - while (it != missingDist.end()) { - if (dist2 > *it) - break; - ++it; - ++itv; - } - missingDist.insert(it, dist2); - missing.insert(itv, p); + continue; } + + const auto rcPos = glm::split(p); + if(auto it_r = cl_area->regionCache.find(rcPos.first); it_r != cl_area->regionCache.end()) { + if (auto it_rc = it_r->second.find(rcPos.second); it_rc != it_r->second.end() && (it_rc->second.swap() || options.useAverages)) { + auto ck = std::make_shared(it_rc->second); + ck->invalidate(geometry::Faces::All); + chunks.emplace(p, std::dynamic_pointer_cast(ck)); + contouring->onNotify(std::make_pair(area.first, p), diff, chunks); + if (ck->isTrusted(options.trustMajorant)) + continue; + } + } + + const region_pos rPos = glm::divide(p); + if (!cl_area->regionCache.contains(rPos)) { + cl_area->regionCache.emplace(rPos, 0); + missingRegions.push_back(rPos); + } + if (missingDist.size() >= net::MAX_PENDING_CHUNK_COUNT) { + if (dist2 > missingDist.front()) + continue; + missingDist.erase(missingDist.begin()); + missingChunks.erase(missingChunks.begin()); + } + + auto it = missingDist.begin(); + auto itv = missingChunks.begin(); + while (it != missingDist.end()) { + if (dist2 > *it) + break; + ++it; + ++itv; + } + missingDist.insert(it, dist2); + missingChunks.insert(itv, p); } }}} - if(!missing.empty()) { - auto packet = net::PacketWriter(net::client_packet_type::MISSING_CHUNKS, sizeof(area_id) + missing.size() * sizeof(chunk_pos)); + if(!missingChunks.empty()) { + auto packet = net::PacketWriter(net::client_packet_type::MISSING_CHUNKS, sizeof(area_id) + missingChunks.size() * sizeof(chunk_pos)); packet.write(area.first); - packet.write(missing.data(), missing.size() * sizeof(chunk_pos)); + packet.write(missingChunks.data(), missingChunks.size() * sizeof(chunk_pos)); + peer.send(packet.finish()); + LOG("Query " << missingChunks.size()); + } + if(!missingRegions.empty()) { + auto packet = net::PacketWriter(net::client_packet_type::MISSING_REGIONS, sizeof(area_id) + missingRegions.size() * sizeof(region_pos)); + packet.write(area.first); + packet.write(missingRegions.data(), missingRegions.size() * sizeof(region_pos)); peer.send(packet.finish()); } mayQueryChunks = false; @@ -159,6 +191,41 @@ bool DistantUniverse::onPacket(const data::out_view& buf, net::PacketFlags) { break; } + case server_packet_type::REGION: { + ZoneScopedN("Region"); + area_ pos; + if(!packet.read(pos)) + break; + + auto it = areas.find(pos.first); + if(it == areas.end()) { + LOG_W("Region area not found " << pos.first.index); + break; + } + auto area = std::dynamic_pointer_cast(it->second); + auto it_r = area->regionCache.find(pos.second); + if (it_r == area->regionCache.end()) { + it_r = area->regionCache.emplace(pos.second, 0).first; + } + // MAYBE: use Voxel swag bit to flag full void/air chunks to avoiding MISSING_CHUNKS + // MAYBE: then create virtual or non chunk of single material + ushort full = 0; + ushort total = 0; + while (!packet.isFull()) { + region_chunk_pos cpos; + Voxel voxel; + if (!packet.read(cpos) || !packet.read(voxel)) + break; + + it_r->second.insert_or_assign(cpos, voxel); + total++; + if (voxel.swap()) + full++; + } + LOG(full << "/" << total); + break; + } + case server_packet_type::CHUNK: { ZoneScopedN("Chunk"); if (!dict.has_value()) diff --git a/src/client/world/DistantUniverse.hpp b/src/client/world/DistantUniverse.hpp index c0a53b4..a8525e5 100644 --- a/src/client/world/DistantUniverse.hpp +++ b/src/client/world/DistantUniverse.hpp @@ -49,8 +49,7 @@ namespace world::client { bool mayQueryChunks = false; uint64_t moveCounter = 0; - ushort loadDistance; - ushort keepDistance; + options options; ushort serverDistance; net::client::Client peer; diff --git a/src/client/world/Universe.hpp b/src/client/world/Universe.hpp index 8b22f1e..8990df5 100644 --- a/src/client/world/Universe.hpp +++ b/src/client/world/Universe.hpp @@ -18,6 +18,14 @@ namespace world::client { Universe(const std::string& contouring); virtual ~Universe(); + /// Options for distant universe + struct options: world::Universe::options { + /// Render temporary partial average as placeholder + bool useAverages = false; + /// Dont query chunks with absolute majorant average + /// Avoid querying whole space but may cause missed edits in those chunks + bool trustMajorant = true; + }; struct connection: net::address { connection(): net::address{"localhost", 4242} { } }; diff --git a/src/core/net/data.hpp b/src/core/net/data.hpp index 3f9a471..1592693 100644 --- a/src/core/net/data.hpp +++ b/src/core/net/data.hpp @@ -37,14 +37,18 @@ enum class server_packet_type: uint8_t { /// List all areas /// {area_id, world::Area::params}[] AREAS = 16, + /// Average of preloaded chunks in region + /// {area_, {region_chunk_pos, Voxel}[]} + /// Voxel swap bit indicate a chunk full of Voxel without flag bit + REGION = 17, /// Full chunk update /// {area_, zstd} /// empty: all sent - CHUNK = 17, + CHUNK = 18, /// Chunk changes /// {area_id, {chunk_pos, ushort(count), Chunk::Edit[]}[]} notify /// MAYBE: compress - EDITS = 18, + EDITS = 19, /// Declare entities types /// {size_t(index), vec3(size), vec3(scale)} @@ -72,6 +76,10 @@ enum class client_packet_type: uint8_t { /// actions::FillShape FILL_SHAPE = 0, + /// Request missing regions + /// area_id, region_pos[] + MISSING_REGIONS = 7, + /// Request missing chunks /// area_id, chunk_pos[max: MAX_PENDING_CHUNK_COUNT] MISSING_CHUNKS = 8, diff --git a/src/core/net/io.hpp b/src/core/net/io.hpp index f63e62e..b9ed5b3 100644 --- a/src/core/net/io.hpp +++ b/src/core/net/io.hpp @@ -43,11 +43,7 @@ public: void* data() { return buffer.writeTo(0); } void reserve(size_t target) { if (target >= buffer.siz - buffer.cur) { - auto old = buffer.ptr; - buffer.ptr = (uint8_t*)malloc(target); - memcpy(buffer.ptr, old, buffer.siz); - free(old); - buffer.siz = target; + buffer.ptr = (uint8_t*)realloc(buffer.ptr, target + buffer.cur); } } void resize(size_t target) { diff --git a/src/core/world/EdittableChunk.cpp b/src/core/world/EdittableChunk.cpp index 08f800d..5f754b4 100644 --- a/src/core/world/EdittableChunk.cpp +++ b/src/core/world/EdittableChunk.cpp @@ -8,6 +8,7 @@ using namespace world::client; EdittableChunk::EdittableChunk(): world::Chunk() { } EdittableChunk::EdittableChunk(std::istream &is): world::Chunk(is) { } +EdittableChunk::EdittableChunk(Voxel val): world::Chunk(), isAverage(true), isMajorant(val.swap()) { voxels.fill(Voxel(val.material(), val.density())); } EdittableChunk::~EdittableChunk() { } std::optional EdittableChunk::update(float deltaTime, bool animate) { diff --git a/src/core/world/EdittableChunk.hpp b/src/core/world/EdittableChunk.hpp index a4a9077..5650da5 100644 --- a/src/core/world/EdittableChunk.hpp +++ b/src/core/world/EdittableChunk.hpp @@ -8,6 +8,8 @@ namespace world::client { class EdittableChunk: public virtual world::Chunk { public: EdittableChunk(std::istream &is); + /// Create from average + EdittableChunk(Voxel val); virtual ~EdittableChunk(); /// Update voxels @@ -28,9 +30,20 @@ namespace world::client { static std::optional getNeighborIdx(chunk_voxel_idx idx, Face dir); + constexpr bool isTrusted(bool allowMajorant) const { return !isAverage || (allowMajorant && isMajorant); } + void unsetMajorant() { + assert(isMajorant); + isMajorant = false; + } + protected: EdittableChunk(); + /// Is temporary average + const bool isAverage = false; + /// Is temporary full valued + bool isMajorant = false; + /// Temporary changes std::vector edits; /// Require update diff --git a/src/core/world/Voxel.hpp b/src/core/world/Voxel.hpp index 2e65ac8..79959d1 100644 --- a/src/core/world/Voxel.hpp +++ b/src/core/world/Voxel.hpp @@ -12,20 +12,25 @@ namespace world { uint16_t value; using material_t = uint_fast16_t; using density_t = uint_fast8_t; - constexpr static const density_t DENSITY_MAX = 0b0111; + constexpr static const density_t DENSITY_MAX = (1 << 3)-1; + constexpr static const material_t MATERIAL_MAX = (1 << 12)-1; + + constexpr static const uint16_t DENSITY_MASK = 0b0111; + constexpr static const uint16_t MATERIAL_MASK = 0b0111'1111'1111'1000; + constexpr static const uint16_t SWAP_MASK = 0b1000'0000'0000'0000; Voxel(uint16_t value = 0): value(value) { } Voxel(material_t material, density_t density, bool swap = false) { assert(density <= DENSITY_MAX); - assert(material < (1 << 12)); - value = (swap & 0b1000'0000'0000'0000) | - ((material << 3) & 0b0111'1111'1111'1000) | - (density & DENSITY_MAX); + assert(material <= MATERIAL_MAX); + value = (swap * SWAP_MASK) | + ((material << 3) & MATERIAL_MASK) | + (density & DENSITY_MASK); } /// Material type constexpr inline material_t material() const { - return (value & 0b0111'1111'1111'1000) >> 3; + return (value & MATERIAL_MASK) >> 3; } /// Texture idx constexpr inline ushort texture() const { @@ -34,7 +39,7 @@ namespace world { /// Quantity of element constexpr inline density_t density() const { - return value & DENSITY_MAX; + return value & DENSITY_MASK; } /// Quantity of element on [0, 1] constexpr inline float density_ratio() const { @@ -44,7 +49,7 @@ namespace world { /// Swap value /// Use external metadata table constexpr inline bool swap() const { - return value & 0b1000'0000'0000'0000; + return (value & SWAP_MASK) != 0; } /// Is solid diff --git a/src/server/world/Chunk.cpp b/src/server/world/Chunk.cpp index f12aaac..26d950c 100644 --- a/src/server/world/Chunk.cpp +++ b/src/server/world/Chunk.cpp @@ -13,13 +13,29 @@ Chunk::Chunk(const chunk_pos& pos, const std::unique_ptr& r Chunk::Chunk(std::istream& str, bool rle): world::Chunk(str, rle) { } Chunk::~Chunk() { } -void Chunk::write(std::ostream& str, bool rle) const { +world::Voxel Chunk::write(std::ostream& str, bool rle) const { + Voxel::material_t majMat = UINT16_MAX; + ushort majCounter = 1; + size_t visibleDensity = 0; + const auto doMaj = [&](const Voxel& current) { + if (current.material() == majMat) { + majCounter++; + } else { + majCounter--; + } + if (majCounter == 0) { + majMat = current.material(); + majCounter = 1; + } + visibleDensity += current.density() * current.is_visible(); + }; if (rle) { const auto *it = voxels.begin(); ushort counter = 1; Voxel current = *it; while(true) { ++it; + doMaj(current); const auto end = (it == voxels.end()); if(end || current.value != it->value) { str.write(reinterpret_cast(&counter), sizeof(counter)); @@ -35,9 +51,14 @@ void Chunk::write(std::ostream& str, bool rle) const { } } else { for(auto current: voxels) { + doMaj(current); str.write(reinterpret_cast(¤t), sizeof(current)); } } + if (majCounter-1 == INT16_MAX) { // Perfect majority + return Voxel(voxels.front().material(), voxels.front().density(), true); + } + return Voxel(majMat, visibleDensity / CHUNK_SIZE); } void Chunk::set(ushort idx, const Voxel& val) { diff --git a/src/server/world/Chunk.hpp b/src/server/world/Chunk.hpp index b4af8bb..98aecc5 100644 --- a/src/server/world/Chunk.hpp +++ b/src/server/world/Chunk.hpp @@ -23,8 +23,9 @@ namespace world::server { /// Is player modified inline bool isModified() const { return modified; } - /// Write to file. - void write(std::ostream& str, bool rle = RLE) const; + /// Write to file and return average (majorant material) + /// Voxel swap bit indicate perfect majority + Voxel write(std::ostream& str, bool rle = RLE) const; protected: Chunk(): world::Chunk() { } diff --git a/src/server/world/Universe.cpp b/src/server/world/Universe.cpp index f324ddb..d1505f8 100644 --- a/src/server/world/Universe.cpp +++ b/src/server/world/Universe.cpp @@ -78,13 +78,25 @@ Universe::Universe(const Universe::options &options): host(options.connection, const auto read_ctx = dicts.make_reader(); const auto write_ctx = dicts.make_writer(); while (running) { - if (std::pair, std::shared_ptr> task; loadQueue.pop(task)) { + if(save_task_t task; saveQueue.pop(task)) { + //NOTE: must always save before load to avoid chunk regen + //MAYBE: queue.take to avoid concurent write or duplicated work on fast move + ZoneScopedN("ProcessSave"); + std::ostringstream out; + if (!task.second.second->isModified()) { + out.setstate(std::ios_base::badbit); + } + const auto average = task.second.second->write(out); + const auto rcPos = glm::split(task.second.first); + const auto reg = task.first.second->getRegion(folderPath, std::make_pair(task.first.first, rcPos.first)); + reg->write(rcPos.second, write_ctx, out.str(), std::make_optional(average)); + } else if (std::pair, std::shared_ptr> task; loadQueue.pop(task)) { //MAYBE: loadQueue.take to avoid duplicated work on fast move ZoneScopedN("ProcessLoad"); const auto &pos = task.first; const auto rcPos = glm::split(pos.second); const auto reg = task.second->getRegion(folderPath, std::make_pair(pos.first, rcPos.first)); - Region::data data; + std::vector data; if(reg->read(rcPos.second, read_ctx, data)) { ZoneScopedN("ProcessRead"); vec_istream idata(data); @@ -94,16 +106,6 @@ Universe::Universe(const Universe::options &options): host(options.connection, ZoneScopedN("ProcessGenerate"); loadedQueue.push({pos, createChunk(pos.second, task.second->getGenerator())}); } - } else if(save_task_t task; saveQueue.pop(task)) { - //MAYBE: queue.take to avoid concurent write or duplicated work on fast move - ZoneScopedN("ProcessSave"); - if(task.second.second->isModified()) { - std::ostringstream out; - task.second.second->write(out); - const auto rcPos = glm::split(task.second.first); - const auto reg = task.first.second->getRegion(folderPath, std::make_pair(task.first.first, rcPos.first)); - reg->write(rcPos.second, write_ctx, out.str()); - } } else { loadQueue.wait(); } @@ -303,6 +305,7 @@ void Universe::update(float deltaTime) { saveQueue.emplace(*it, std::make_pair(it_c->first, std::dynamic_pointer_cast(it_c->second))); it_c = chunks.erase(it_c); } + loadQueue.notify_all(); LOG_I("Unload area " << it->first.index); [[maybe_unused]] auto ok = far_areas.put(it->first, it->second->getParams()); @@ -532,6 +535,42 @@ bool Universe::onPacket(net::server::Peer *peer, const data::out_view &buf, net: + ": " + std::string((const char*)ref.data(), ref.size())); break; } + case client_packet_type::MISSING_REGIONS: { + auto data = peer->getCtx(); + if (auto player = findEntity(PLAYER_ENTITY_ID, data->instanceId)) { + const auto pos = player->pos.as_voxel(); + + area_id id; + if (!packet.read(id)) + break; + if (auto area = areas.find(id); area != areas.end()) { + const chunk_pos areaOffset = glm::divide(pos - area->second->getOffset().as_voxel()); + while (!packet.isFull()) { + region_pos rpos; + if (!packet.read(rpos)) + break; + + if (auto it_r = area->second->getRegions()->find(rpos); it_r != area->second->getRegions()->end()) { + if (glm::length2(areaOffset - glm::lvec3(it_r->first) * glm::lvec3(REGION_LENGTH)) <= glm::pow2(loadDistance + REGION_LENGTH * 2)) { + auto packet = PacketWriter(server_packet_type::REGION, sizeof(area_)); + packet.write>(std::make_pair(id, rpos)); + { + auto vec = packet.varying(); + it_r->second->averages(vec); + } + peer->send(packet.finish(), net::server::CHUNK); + } else { + LOG_T("Request out of range region"); + } + + } + } + } else { + LOG_T("Bad region request"); + } + } + break; + } case client_packet_type::MISSING_CHUNKS: { auto data = peer->getCtx(); if (auto player = findEntity(PLAYER_ENTITY_ID, data->instanceId )) { diff --git a/src/server/world/region/File.cpp b/src/server/world/region/File.cpp index d91c968..cb450be 100644 --- a/src/server/world/region/File.cpp +++ b/src/server/world/region/File.cpp @@ -34,17 +34,23 @@ void FileRegion::load() { file.read(reinterpret_cast(&pos.y), sizeof(region_chunk_pos::value_type)); file.read(reinterpret_cast(&pos.z), sizeof(region_chunk_pos::value_type)); - //NOTE: align uchar pos - if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) { - file.ignore(1); + std::optional avg; + Flags flags = Flags::ZERO; + file.read(reinterpret_cast(&flags), 1); + if (flags & Flags::HAS_AVERAGE) { + Voxel v; + file.read(reinterpret_cast(&v), sizeof(v)); + avg = v; } // Read size ushort size = 0; - file.read(reinterpret_cast(&size), sizeof(size)); + if (!(flags & Flags::EMPTY)) { + file.read(reinterpret_cast(&size), sizeof(size)); + } // Ignore content - if(!index.insert({pos, std::make_pair(size, file.tellg())}).second) { + if(!index.emplace(pos, node{avg, size, file.tellg()}).second) { LOG_E("Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z); } file.ignore(size); @@ -57,16 +63,16 @@ void FileRegion::load() { assert(index.size() == chunkCount); } -bool FileRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, data& out) { +bool FileRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, std::vector& out) { std::unique_lock lock(mutex); const auto it = index.find(pos); - if (it == index.end()) + if (it == index.end() || it->second.size == 0) return false; - data in; - in.resize(it->second.first); - file.seekg(it->second.second); + std::vector in; + in.resize(it->second.size); + file.seekg(it->second.offset); file.read(in.data(), in.size()); if (auto err = ctx.decompress(in, out)) { @@ -82,17 +88,20 @@ bool FileRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, da } return true; } -void FileRegion::write(const region_chunk_pos& pos, const zstd::write_ctx& ctx, const std::string_view& in) { - auto buffer = std::make_unique(); - if (auto err = ctx.compress(in, *buffer)) { - LOG_E("Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " " - << err.value()); - return; +void FileRegion::write(const region_chunk_pos& pos, const zstd::write_ctx& ctx, const std::string_view& in, const std::optional& avg) { + std::unique_ptr> buffer = nullptr; + if (!in.empty()) { + buffer = std::make_unique>(); + if (auto err = ctx.compress(in, *buffer)) { + LOG_E("Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " " + << err.value()); + return; + } } - save({{pos, std::move(buffer)}}); + save(std::make_optional(to_save{pos, std::move(buffer), avg})); } -void FileRegion::save(std::optional>> added) { +void FileRegion::save(std::optional added) { std::unique_lock lock(mutex); const auto tmpPath = path + ".tmp"; @@ -108,7 +117,7 @@ void FileRegion::save(std::optional(&size), sizeof(size)); } - data tmp; + std::vector tmp; for(const auto& chunk: index) { { // Write pos region_chunk_pos pos = chunk.first; @@ -117,42 +126,58 @@ void FileRegion::save(std::optional(&pos.z), sizeof(region_chunk_pos::value_type)); } - //NOTE: align uchar pos - if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) { - tmpFile.put(0); - //MAYBE: store usefull uchar flags + // Write average if present + Flags flags = Flags::ZERO; + if (chunk.second.average.has_value()) + flags = (Flags)(flags | Flags::HAS_AVERAGE); + if (chunk.second.size == 0) + flags = (Flags)(flags | Flags::EMPTY); + tmpFile.write(reinterpret_cast(&flags), 1); + if (flags & Flags::HAS_AVERAGE) { + Voxel v = chunk.second.average.value(); + tmpFile.write(reinterpret_cast(&v), sizeof(v)); } - // Write size - auto size = chunk.second.first; - tmpFile.write(reinterpret_cast(&size), sizeof(size)); + if (!(flags & Flags::EMPTY)) { + // Write size + auto size = chunk.second.size; + tmpFile.write(reinterpret_cast(&size), sizeof(size)); - // Write content - tmp.resize(size); - file.seekg(chunk.second.second); - file.read(tmp.data(), size); - tmpFile.write(tmp.data(), size); + // Write content + tmp.resize(size); + file.seekg(chunk.second.offset); + file.read(tmp.data(), size); + tmpFile.write(tmp.data(), size); + } } if(added.has_value()) { { // Write pos - region_chunk_pos pos = added.value().first; + region_chunk_pos pos = added.value().pos; tmpFile.write(reinterpret_cast(&pos.x), sizeof(region_chunk_pos::value_type)); tmpFile.write(reinterpret_cast(&pos.y), sizeof(region_chunk_pos::value_type)); tmpFile.write(reinterpret_cast(&pos.z), sizeof(region_chunk_pos::value_type)); } - //NOTE: align uchar pos - if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) { - tmpFile.put(0); - //MAYBE: store usefull uchar flags + // Write average if present + Flags flags = Flags::ZERO; + if (added.value().average.has_value()) + flags = (Flags)(flags | Flags::HAS_AVERAGE); + if (added.value().data == nullptr) + flags = (Flags)(flags | Flags::EMPTY); + tmpFile.write(reinterpret_cast(&flags), 1); + if (flags & Flags::HAS_AVERAGE) { + Voxel v = added.value().average.value(); + tmpFile.write(reinterpret_cast(&v), sizeof(v)); } - // Write size - auto size = added.value().second->size(); - tmpFile.write(reinterpret_cast(&size), sizeof(size)); + if (!(flags & Flags::EMPTY)) { + // Write size + auto size = added.value().data->size(); + tmpFile.write(reinterpret_cast(&size), sizeof(size)); - // Write content - tmpFile.write(added.value().second->data(), size); + // Write content + tmpFile.write(added.value().data->data(), size); + } } if (!tmpFile.good()) { diff --git a/src/server/world/region/File.hpp b/src/server/world/region/File.hpp index 3fbf92d..6cd1d51 100644 --- a/src/server/world/region/File.hpp +++ b/src/server/world/region/File.hpp @@ -4,6 +4,8 @@ #include #include #include "../../../core/world/forward.h" +#include "../../../core/world/Voxel.hpp" +#include "../../../core/data/mem.hpp" #include "../../../core/utils/zctx.hpp" #include "../../../core/data/math.hpp" @@ -14,20 +16,49 @@ namespace world::server { FileRegion(const std::string& folderPath, const area_ &pos); ~FileRegion(); - typedef std::vector data; - - bool read(const region_chunk_pos &pos, const zstd::read_ctx& ctx, data &out); - void write(const region_chunk_pos &pos, const zstd::write_ctx& ctx, const std::string_view &in); + template + void averages(D &out) { + std::shared_lock lock(mutex); + out.resize(index.size() * (sizeof(region_chunk_pos) + sizeof(Voxel))); + data::in_view in((uint8_t*)out.data(), out.size()); + for(const auto& r: index) { + if (r.second.average.has_value()) { + in.write((const uint8_t*)&r.first, sizeof(r.first)); + in.write((const uint8_t*)&r.second.average.value(), sizeof(r.second.average.value())); + } + } + out.resize(in.cur); + } + bool read(const region_chunk_pos &pos, const zstd::read_ctx& ctx, std::vector &out); + void write(const region_chunk_pos &pos, const zstd::write_ctx& ctx, const std::string_view &in, const std::optional& average); private: - void save(std::optional>> added); + struct to_save { + region_chunk_pos pos; + std::unique_ptr> data; + std::optional average; + }; + void save(std::optional added); std::string path; //TODO: use tickets to remove unused regions + enum Flags: unsigned char { + ZERO = 0, + HAS_AVERAGE = 1 << 0, + EMPTY = 1 << 1 + }; + std::shared_mutex mutex; std::ifstream file; - robin_hood::unordered_map> index; + struct node { + node(const std::optional& a, ushort s, std::streampos o): + average(a), size(s), offset(o) { } + std::optional average; + ushort size; + std::streampos offset; + }; + robin_hood::unordered_map index; void load(); }; diff --git a/src/server/world/region/Memory.cpp b/src/server/world/region/Memory.cpp index fda5be7..3eff23c 100644 --- a/src/server/world/region/Memory.cpp +++ b/src/server/world/region/Memory.cpp @@ -14,13 +14,6 @@ MemoryRegion::MemoryRegion(const std::string &folderPath, const area_second; - it = content.erase(it); - } } void MemoryRegion::load() { @@ -43,20 +36,30 @@ void MemoryRegion::load() { file.read(reinterpret_cast(&pos.y), sizeof(region_chunk_pos::value_type)); file.read(reinterpret_cast(&pos.z), sizeof(region_chunk_pos::value_type)); - //NOTE: align uchar pos - if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) { - file.ignore(1); + // Read average if present + std::optional avg; + Flags flags = Flags::ZERO; + file.read(reinterpret_cast(&flags), 1); + if (flags & Flags::HAS_AVERAGE) { + Voxel v; + file.read(reinterpret_cast(&v), sizeof(v)); + avg = v; } // Read size ushort size = 0; - file.read(reinterpret_cast(&size), sizeof(size)); + if (!(flags & Flags::EMPTY)) { + file.read(reinterpret_cast(&size), sizeof(size)); + } // Read content - const auto data = new MemoryRegion::data(); - data->resize(size); - file.read(data->data(), data->size()); - if(!content.insert({pos, data}).second) { + std::unique_ptr> data = nullptr; + if (size > 0) { + data = std::make_unique>(); + data->resize(size); + file.read(data->data(), data->size()); + } + if(!content.emplace(pos, node(avg, std::move(data))).second) { LOG_E("Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z); } file.peek(); @@ -68,14 +71,14 @@ void MemoryRegion::load() { assert(content.size() == chunkCount); file.close(); } -bool MemoryRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, data& out) { +bool MemoryRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, std::vector& out) { std::shared_lock lock(mutex); const auto it = content.find(pos); - if (it == content.end()) + if (it == content.end() || it->second.data == nullptr) return false; - if(auto err = ctx.decompress(*it->second, out)) { + if(auto err = ctx.decompress(*it->second.data.get(), out)) { LOG_E("Corrupted region chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " " << err.value()); #ifdef REMOVE_CORRUPTED @@ -91,26 +94,28 @@ bool MemoryRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, } return true; } -void MemoryRegion::write(const region_chunk_pos& pos, const zstd::write_ctx& ctx, const std::string_view& in) { - const auto buffer = new MemoryRegion::data(); - - if (auto err = ctx.compress(in, *buffer)) { - LOG_E("Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " " - << err.value()); - return; +void MemoryRegion::write(const region_chunk_pos& pos, const zstd::write_ctx& ctx, const std::string_view& in, const std::optional& avg) { + std::unique_ptr> buffer = nullptr; + if (!in.empty()) { + buffer = std::make_unique>(); + if (auto err = ctx.compress(in, *buffer.get())) { + LOG_E("Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " " + << err.value()); + return; + } } - { std::unique_lock lock(mutex); // Save buffer const auto it = content.find(pos); - if (it != content.end()) - { - delete it->second; + if (it != content.end()) { + if (buffer == nullptr) { + buffer = std::move(it->second.data); + } content.erase(it); } - content.insert({pos, buffer}); + content.emplace(pos, node(avg, std::move(buffer))); changed = true; } save(false); @@ -134,10 +139,6 @@ void MemoryRegion::save(bool force) { } for(const auto& chunk: content) { - assert(chunk.second->size() < USHRT_MAX); - auto size = (ushort)chunk.second->size(); - const auto out = chunk.second->data(); - { // Write pos region_chunk_pos pos = chunk.first; file.write(reinterpret_cast(&pos.x), sizeof(region_chunk_pos::value_type)); @@ -145,17 +146,29 @@ void MemoryRegion::save(bool force) { file.write(reinterpret_cast(&pos.z), sizeof(region_chunk_pos::value_type)); } - //NOTE: align uchar pos - if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) { - file.put(0); - //MAYBE: store usefull uchar flags + // Write average if present + Flags flags = Flags::ZERO; + if (chunk.second.average.has_value()) + flags = (Flags)(flags | Flags::HAS_AVERAGE); + if (chunk.second.data == nullptr) + flags = (Flags)(flags | Flags::EMPTY); + file.write(reinterpret_cast(&flags), sizeof(flags)); + if (flags & Flags::HAS_AVERAGE) { + Voxel v = chunk.second.average.value(); + file.write(reinterpret_cast(&v), sizeof(v)); } - // Write size - file.write(reinterpret_cast(&size), sizeof(size)); + if (!(flags & Flags::EMPTY)) { + assert(chunk.second.data->size() < USHRT_MAX); + auto size = (ushort)chunk.second.data->size(); + const auto out = chunk.second.data->data(); - // Write content - file.write(out, size); + // Write size + file.write(reinterpret_cast(&size), sizeof(size)); + + // Write content + file.write(out, size); + } } if (!file.good()) { diff --git a/src/server/world/region/Memory.hpp b/src/server/world/region/Memory.hpp index 689ec1a..96c0d30 100644 --- a/src/server/world/region/Memory.hpp +++ b/src/server/world/region/Memory.hpp @@ -4,6 +4,8 @@ #include #include #include "../../../core/world/forward.h" +#include "../../../core/world/Voxel.hpp" +#include "../../../core/data/mem.hpp" #include "../../../core/utils/zctx.hpp" #include "../../../core/data/math.hpp" @@ -14,10 +16,21 @@ namespace world::server { MemoryRegion(const std::string& folderPath, const area_ &pos); ~MemoryRegion(); - typedef std::vector data; - - bool read(const region_chunk_pos &pos, const zstd::read_ctx& ctx, data &out); - void write(const region_chunk_pos &pos, const zstd::write_ctx& ctx, const std::string_view &in); + template + void averages(D &out) { + std::shared_lock lock(mutex); + out.resize(content.size() * (sizeof(region_chunk_pos) + sizeof(Voxel))); + data::in_view in((uint8_t*)out.data(), out.size()); + for(const auto& r: content) { + if (r.second.average.has_value()) { + in.write((const uint8_t*)&r.first, sizeof(r.first)); + in.write((const uint8_t*)&r.second.average.value(), sizeof(r.second.average.value())); + } + } + out.resize(in.cur); + } + bool read(const region_chunk_pos &pos, const zstd::read_ctx &ctx, std::vector &out); + void write(const region_chunk_pos &pos, const zstd::write_ctx& ctx, const std::string_view &in, const std::optional& average); private: void save(bool force = true); @@ -25,8 +38,20 @@ namespace world::server { std::string path; //TODO: use tickets to remove unused regions + enum Flags: unsigned char { + ZERO = 0, + HAS_AVERAGE = 1 << 0, + EMPTY = 1 << 1 + }; + std::shared_mutex mutex; - robin_hood::unordered_map content; + struct node { + node(const std::optional& a, std::unique_ptr> d): + average(a), data(std::move(d)) { } + std::optional average; + std::unique_ptr> data; + }; + robin_hood::unordered_map content; bool changed = false; void load();