me
/
Univerxel
1
0
Fork 0
Code Releases Activity

Regions level preload (1:32)

windows
May B. 2020-11-06 17:32:00 +01:00
parent f5c46d5b46
commit 2bb02bef1f
17 changed files with 412 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/client/config.hpp
View File

@ -64,6 +64,8 @@ public:
world.loadDistance = config["world"]["load_distance"].value_or(world.loadDistance);
world.keepDistance = config["world"]["keep_distance"].value_or(world.keepDistance);
world.useAverages = config["world"]["use_averages"].value_or(world.useAverages);
world.trustMajorant = config["world"]["trust_majorant"].value_or(world.trustMajorant);
voxel_density = config["world"]["voxel_density"].value_or(voxel_density);
#ifndef STANDALONE
@ -140,6 +142,8 @@ public:
config.insert_or_assign("world", toml::table({
{"load_distance", world.loadDistance},
{"keep_distance", world.keepDistance},
{"use_averages", world.useAverages},
{"trust_majorant", world.trustMajorant},
{"voxel_density", voxel_density}
}));
if(connection.has_value()) {

2
src/client/world/Area.hpp
View File

@ -16,6 +16,8 @@ namespace world::client {
curvature = p.curvature;
}
robin_hood::unordered_map<region_pos, robin_hood::unordered_map<region_chunk_pos, Voxel>> regionCache;
private:
std::optional<double> curvature;
};

123
src/client/world/DistantUniverse.cpp
View File

@ -11,8 +11,8 @@
using namespace world::client;
DistantUniverse::DistantUniverse(const connection& ct, const options& opt, const std::string& contouring): Universe(contouring),
loadDistance(opt.loadDistance), keepDistance(opt.keepDistance), serverDistance(0),
DistantUniverse::DistantUniverse(const connection& ct, const Universe::options& opt, const std::string& contouring):
Universe(contouring), options(opt), serverDistance(0),
peer(ct, [&](const data::out_view& buf, net::PacketFlags flags){ return onPacket(buf, flags); }) { }
DistantUniverse::~DistantUniverse() { }
@ -33,11 +33,11 @@ void DistantUniverse::update(voxel_pos pos, float deltaTime) {
const bool chunkChangeArea = (false && area.second->move(glm::vec3(deltaTime))) || chunkChange; // TODO: area.velocity
const chunk_pos diff = glm::divide(pos - area.second->getOffset().as_voxel());
auto &chunks = area.second->setChunks();
if (glm::length2(diff) <= glm::pow2(keepDistance + area.second->getChunks().getRadius())) {
if (glm::length2(diff) <= glm::pow2(options.keepDistance + area.second->getChunks().getRadius())) {
ZoneScopedN("Alive");
auto it_c = chunks.begin();
while (it_c != chunks.end()) {
if (glm::length2(diff - it_c->first) > glm::pow2(keepDistance)) {
if (glm::length2(diff - it_c->first) > glm::pow2(options.keepDistance)) {
it_c = chunks.erase(it_c);
} else {
if(const auto neighbors = std::dynamic_pointer_cast<world::client::EdittableChunk>(it_c->second)->update(deltaTime, true /*MAYBE: random update*/)) {
@ -46,45 +46,77 @@ void DistantUniverse::update(voxel_pos pos, float deltaTime) {
++it_c;
}
}
//FIXME: Produce important duplicated (chunkChangeArea)
//MAYBE: if dist to unloaded chunk < X
if (chunkChangeArea || mayQueryChunks) { // Request missing chunks
ZoneScopedN("Missing");
std::vector<chunk_pos> missing;
std::vector<chunk_pos> missingChunks;
std::vector<region_pos> missingRegions;
std::vector<long> missingDist;
const auto cl_area = std::dynamic_pointer_cast<Area>(area.second);
//TODO: use easy sphere fill
const int queryDistance = std::min(loadDistance, serverDistance);
const int queryDistance = std::min(options.loadDistance, serverDistance);
for (int x = -queryDistance; x <= queryDistance; x++) {
for (int y = -queryDistance; y <= queryDistance; y++) {
for (int z = -queryDistance; z <= queryDistance; z++) {
const auto dist2 = x * x + y * y + z * z;
const auto p = diff + chunk_pos(x, y, z);
if (dist2 <= queryDistance * queryDistance && chunks.inRange(p)) {
if (chunks.find(p) != chunks.end()) {
if (auto it = chunks.find(p); it != chunks.end() &&
std::dynamic_pointer_cast<world::client::EdittableChunk>(it->second)->isTrusted(options.trustMajorant))
//TODO: config accept perfect average
{
contouring->onNotify(std::make_pair(area.first, p), diff, chunks);
} else {
if (missingDist.size() >= net::MAX_PENDING_CHUNK_COUNT) {
if (dist2 > missingDist.front())
continue;
missingDist.erase(missingDist.begin());
missing.erase(missing.begin());
}
auto it = missingDist.begin();
auto itv = missing.begin();
while (it != missingDist.end()) {
if (dist2 > *it)
break;
++it;
++itv;
}
missingDist.insert(it, dist2);
missing.insert(itv, p);
continue;
}
const auto rcPos = glm::split(p);
if(auto it_r = cl_area->regionCache.find(rcPos.first); it_r != cl_area->regionCache.end()) {
if (auto it_rc = it_r->second.find(rcPos.second); it_rc != it_r->second.end() && (it_rc->second.swap() || options.useAverages)) {
auto ck = std::make_shared<world::client::EdittableChunk>(it_rc->second);
ck->invalidate(geometry::Faces::All);
chunks.emplace(p, std::dynamic_pointer_cast<world::Chunk>(ck));
contouring->onNotify(std::make_pair(area.first, p), diff, chunks);
if (ck->isTrusted(options.trustMajorant))
continue;
}
}
const region_pos rPos = glm::divide(p);
if (!cl_area->regionCache.contains(rPos)) {
cl_area->regionCache.emplace(rPos, 0);
missingRegions.push_back(rPos);
}
if (missingDist.size() >= net::MAX_PENDING_CHUNK_COUNT) {
if (dist2 > missingDist.front())
continue;
missingDist.erase(missingDist.begin());
missingChunks.erase(missingChunks.begin());
}
auto it = missingDist.begin();
auto itv = missingChunks.begin();
while (it != missingDist.end()) {
if (dist2 > *it)
break;
++it;
++itv;
}
missingDist.insert(it, dist2);
missingChunks.insert(itv, p);
}
}}}
if(!missing.empty()) {
auto packet = net::PacketWriter(net::client_packet_type::MISSING_CHUNKS, sizeof(area_id) + missing.size() * sizeof(chunk_pos));
if(!missingChunks.empty()) {
auto packet = net::PacketWriter(net::client_packet_type::MISSING_CHUNKS, sizeof(area_id) + missingChunks.size() * sizeof(chunk_pos));
packet.write(area.first);
packet.write(missing.data(), missing.size() * sizeof(chunk_pos));
packet.write(missingChunks.data(), missingChunks.size() * sizeof(chunk_pos));
peer.send(packet.finish());
LOG("Query " << missingChunks.size());
}
if(!missingRegions.empty()) {
auto packet = net::PacketWriter(net::client_packet_type::MISSING_REGIONS, sizeof(area_id) + missingRegions.size() * sizeof(region_pos));
packet.write(area.first);
packet.write(missingRegions.data(), missingRegions.size() * sizeof(region_pos));
peer.send(packet.finish());
}
mayQueryChunks = false;
@ -159,6 +191,41 @@ bool DistantUniverse::onPacket(const data::out_view& buf, net::PacketFlags) {
break;
}
case server_packet_type::REGION: {
ZoneScopedN("Region");
area_<region_pos> pos;
if(!packet.read(pos))
break;
auto it = areas.find(pos.first);
if(it == areas.end()) {
LOG_W("Region area not found " << pos.first.index);
break;
}
auto area = std::dynamic_pointer_cast<world::client::Area>(it->second);
auto it_r = area->regionCache.find(pos.second);
if (it_r == area->regionCache.end()) {
it_r = area->regionCache.emplace(pos.second, 0).first;
}
// MAYBE: use Voxel swag bit to flag full void/air chunks to avoiding MISSING_CHUNKS
// MAYBE: then create virtual or non chunk of single material
ushort full = 0;
ushort total = 0;
while (!packet.isFull()) {
region_chunk_pos cpos;
Voxel voxel;
if (!packet.read(cpos) || !packet.read(voxel))
break;
it_r->second.insert_or_assign(cpos, voxel);
total++;
if (voxel.swap())
full++;
}
LOG(full << "/" << total);
break;
}
case server_packet_type::CHUNK: {
ZoneScopedN("Chunk");
if (!dict.has_value())

3
src/client/world/DistantUniverse.hpp
View File

@ -49,8 +49,7 @@ namespace world::client {
bool mayQueryChunks = false;
uint64_t moveCounter = 0;
ushort loadDistance;
ushort keepDistance;
options options;
ushort serverDistance;
net::client::Client peer;

8
src/client/world/Universe.hpp
View File

@ -18,6 +18,14 @@ namespace world::client {
Universe(const std::string& contouring);
virtual ~Universe();
/// Options for distant universe
struct options: world::Universe::options {
/// Render temporary partial average as placeholder
bool useAverages = false;
/// Dont query chunks with absolute majorant average
/// Avoid querying whole space but may cause missed edits in those chunks
bool trustMajorant = true;
};
struct connection: net::address {
connection(): net::address{"localhost", 4242} { }
};

12
src/core/net/data.hpp
View File

@ -37,14 +37,18 @@ enum class server_packet_type: uint8_t {
/// List all areas
/// {area_id, world::Area::params}[]
AREAS = 16,
/// Average of preloaded chunks in region
/// {area_<region_pos>, {region_chunk_pos, Voxel}[]}
/// Voxel swap bit indicate a chunk full of Voxel without flag bit
REGION = 17,
/// Full chunk update
/// {area_<chunk_pos>, zstd<chunk rle>}
/// empty: all sent
CHUNK = 17,
CHUNK = 18,
/// Chunk changes
/// {area_id, {chunk_pos, ushort(count), Chunk::Edit[]}[]} notify
/// MAYBE: compress
EDITS = 18,
EDITS = 19,
/// Declare entities types
/// {size_t(index), vec3(size), vec3(scale)}
@ -72,6 +76,10 @@ enum class client_packet_type: uint8_t {
/// actions::FillShape
FILL_SHAPE = 0,
/// Request missing regions
/// area_id, region_pos[]
MISSING_REGIONS = 7,
/// Request missing chunks
/// area_id, chunk_pos[max: MAX_PENDING_CHUNK_COUNT]
MISSING_CHUNKS = 8,

6
src/core/net/io.hpp
View File

@ -43,11 +43,7 @@ public:
void* data() { return buffer.writeTo(0); }
void reserve(size_t target) {
if (target >= buffer.siz - buffer.cur) {
auto old = buffer.ptr;
buffer.ptr = (uint8_t*)malloc(target);
memcpy(buffer.ptr, old, buffer.siz);
free(old);
buffer.siz = target;
buffer.ptr = (uint8_t*)realloc(buffer.ptr, target + buffer.cur);
}
}
void resize(size_t target) {

1
src/core/world/EdittableChunk.cpp
View File

@ -8,6 +8,7 @@ using namespace world::client;
EdittableChunk::EdittableChunk(): world::Chunk() { }
EdittableChunk::EdittableChunk(std::istream &is): world::Chunk(is) { }
EdittableChunk::EdittableChunk(Voxel val): world::Chunk(), isAverage(true), isMajorant(val.swap()) { voxels.fill(Voxel(val.material(), val.density())); }
EdittableChunk::~EdittableChunk() { }
std::optional<Faces> EdittableChunk::update(float deltaTime, bool animate) {

13
src/core/world/EdittableChunk.hpp
View File

@ -8,6 +8,8 @@ namespace world::client {
class EdittableChunk: public virtual world::Chunk {
public:
EdittableChunk(std::istream &is);
/// Create from average
EdittableChunk(Voxel val);
virtual ~EdittableChunk();
/// Update voxels
@ -28,9 +30,20 @@ namespace world::client {
static std::optional<chunk_voxel_idx> getNeighborIdx(chunk_voxel_idx idx, Face dir);
constexpr bool isTrusted(bool allowMajorant) const { return !isAverage || (allowMajorant && isMajorant); }
void unsetMajorant() {
assert(isMajorant);
isMajorant = false;
}
protected:
EdittableChunk();
/// Is temporary average
const bool isAverage = false;
/// Is temporary full valued
bool isMajorant = false;
/// Temporary changes
std::vector<Chunk::Edit> edits;
/// Require update

21
src/core/world/Voxel.hpp
View File

@ -12,20 +12,25 @@ namespace world {
uint16_t value;
using material_t = uint_fast16_t;
using density_t = uint_fast8_t;
constexpr static const density_t DENSITY_MAX = 0b0111;
constexpr static const density_t DENSITY_MAX = (1 << 3)-1;
constexpr static const material_t MATERIAL_MAX = (1 << 12)-1;
constexpr static const uint16_t DENSITY_MASK = 0b0111;
constexpr static const uint16_t MATERIAL_MASK = 0b0111'1111'1111'1000;
constexpr static const uint16_t SWAP_MASK = 0b1000'0000'0000'0000;
Voxel(uint16_t value = 0): value(value) { }
Voxel(material_t material, density_t density, bool swap = false) {
assert(density <= DENSITY_MAX);
assert(material < (1 << 12));
value = (swap & 0b1000'0000'0000'0000) |
((material << 3) & 0b0111'1111'1111'1000) |
(density & DENSITY_MAX);
assert(material <= MATERIAL_MAX);
value = (swap * SWAP_MASK) |
((material << 3) & MATERIAL_MASK) |
(density & DENSITY_MASK);
}
/// Material type
constexpr inline material_t material() const {
return (value & 0b0111'1111'1111'1000) >> 3;
return (value & MATERIAL_MASK) >> 3;
}
/// Texture idx
constexpr inline ushort texture() const {
@ -34,7 +39,7 @@ namespace world {
/// Quantity of element
constexpr inline density_t density() const {
return value & DENSITY_MAX;
return value & DENSITY_MASK;
}
/// Quantity of element on [0, 1]
constexpr inline float density_ratio() const {
@ -44,7 +49,7 @@ namespace world {
/// Swap value
/// Use external metadata table
constexpr inline bool swap() const {
return value & 0b1000'0000'0000'0000;
return (value & SWAP_MASK) != 0;
}
/// Is solid

23
src/server/world/Chunk.cpp
View File

@ -13,13 +13,29 @@ Chunk::Chunk(const chunk_pos& pos, const std::unique_ptr<generator::Abstract>& r
Chunk::Chunk(std::istream& str, bool rle): world::Chunk(str, rle) { }
Chunk::~Chunk() { }
void Chunk::write(std::ostream& str, bool rle) const {
world::Voxel Chunk::write(std::ostream& str, bool rle) const {
Voxel::material_t majMat = UINT16_MAX;
ushort majCounter = 1;
size_t visibleDensity = 0;
const auto doMaj = [&](const Voxel& current) {
if (current.material() == majMat) {
majCounter++;
} else {
majCounter--;
}
if (majCounter == 0) {
majMat = current.material();
majCounter = 1;
}
visibleDensity += current.density() * current.is_visible();
};
if (rle) {
const auto *it = voxels.begin();
ushort counter = 1;
Voxel current = *it;
while(true) {
++it;
doMaj(current);
const auto end = (it == voxels.end());
if(end || current.value != it->value) {
str.write(reinterpret_cast<char *>(&counter), sizeof(counter));
@ -35,9 +51,14 @@ void Chunk::write(std::ostream& str, bool rle) const {
}
} else {
for(auto current: voxels) {
doMaj(current);
str.write(reinterpret_cast<char *>(&current), sizeof(current));
}
}
if (majCounter-1 == INT16_MAX) { // Perfect majority
return Voxel(voxels.front().material(), voxels.front().density(), true);
}
return Voxel(majMat, visibleDensity / CHUNK_SIZE);
}
void Chunk::set(ushort idx, const Voxel& val) {

5
src/server/world/Chunk.hpp
View File

@ -23,8 +23,9 @@ namespace world::server {
/// Is player modified
inline bool isModified() const { return modified; }
/// Write to file.
void write(std::ostream& str, bool rle = RLE) const;
/// Write to file and return average (majorant material)
/// Voxel swap bit indicate perfect majority
Voxel write(std::ostream& str, bool rle = RLE) const;
protected:
Chunk(): world::Chunk() { }

63
src/server/world/Universe.cpp
View File

@ -78,13 +78,25 @@ Universe::Universe(const Universe::options &options): host(options.connection,
const auto read_ctx = dicts.make_reader();
const auto write_ctx = dicts.make_writer();
while (running) {
if (std::pair<area_<chunk_pos>, std::shared_ptr<Area>> task; loadQueue.pop(task)) {
if(save_task_t task; saveQueue.pop(task)) {
//NOTE: must always save before load to avoid chunk regen
//MAYBE: queue.take to avoid concurent write or duplicated work on fast move
ZoneScopedN("ProcessSave");
std::ostringstream out;
if (!task.second.second->isModified()) {
out.setstate(std::ios_base::badbit);
}
const auto average = task.second.second->write(out);
const auto rcPos = glm::split(task.second.first);
const auto reg = task.first.second->getRegion(folderPath, std::make_pair(task.first.first, rcPos.first));
reg->write(rcPos.second, write_ctx, out.str(), std::make_optional(average));
} else if (std::pair<area_<chunk_pos>, std::shared_ptr<Area>> task; loadQueue.pop(task)) {
//MAYBE: loadQueue.take to avoid duplicated work on fast move
ZoneScopedN("ProcessLoad");
const auto &pos = task.first;
const auto rcPos = glm::split(pos.second);
const auto reg = task.second->getRegion(folderPath, std::make_pair(pos.first, rcPos.first));
Region::data data;
std::vector<char> data;
if(reg->read(rcPos.second, read_ctx, data)) {
ZoneScopedN("ProcessRead");
vec_istream idata(data);
@ -94,16 +106,6 @@ Universe::Universe(const Universe::options &options): host(options.connection,
ZoneScopedN("ProcessGenerate");
loadedQueue.push({pos, createChunk(pos.second, task.second->getGenerator())});
}
} else if(save_task_t task; saveQueue.pop(task)) {
//MAYBE: queue.take to avoid concurent write or duplicated work on fast move
ZoneScopedN("ProcessSave");
if(task.second.second->isModified()) {
std::ostringstream out;
task.second.second->write(out);
const auto rcPos = glm::split(task.second.first);
const auto reg = task.first.second->getRegion(folderPath, std::make_pair(task.first.first, rcPos.first));
reg->write(rcPos.second, write_ctx, out.str());
}
} else {
loadQueue.wait();
}
@ -303,6 +305,7 @@ void Universe::update(float deltaTime) {
saveQueue.emplace(*it, std::make_pair(it_c->first, std::dynamic_pointer_cast<Chunk>(it_c->second)));
it_c = chunks.erase(it_c);
}
loadQueue.notify_all();
LOG_I("Unload area " << it->first.index);
[[maybe_unused]]
auto ok = far_areas.put(it->first, it->second->getParams());
@ -532,6 +535,42 @@ bool Universe::onPacket(net::server::Peer *peer, const data::out_view &buf, net:
+ ": " + std::string((const char*)ref.data(), ref.size()));
break;
}
case client_packet_type::MISSING_REGIONS: {
auto data = peer->getCtx<net_client>();
if (auto player = findEntity(PLAYER_ENTITY_ID, data->instanceId)) {
const auto pos = player->pos.as_voxel();
area_id id;
if (!packet.read<area_id>(id))
break;
if (auto area = areas.find(id); area != areas.end()) {
const chunk_pos areaOffset = glm::divide(pos - area->second->getOffset().as_voxel());
while (!packet.isFull()) {
region_pos rpos;
if (!packet.read(rpos))
break;
if (auto it_r = area->second->getRegions()->find(rpos); it_r != area->second->getRegions()->end()) {
if (glm::length2(areaOffset - glm::lvec3(it_r->first) * glm::lvec3(REGION_LENGTH)) <= glm::pow2(loadDistance + REGION_LENGTH * 2)) {
auto packet = PacketWriter(server_packet_type::REGION, sizeof(area_<region_pos>));
packet.write<area_<region_pos>>(std::make_pair(id, rpos));
{
auto vec = packet.varying();
it_r->second->averages(vec);
}
peer->send(packet.finish(), net::server::CHUNK);
} else {
LOG_T("Request out of range region");
}
}
}
} else {
LOG_T("Bad region request");
}
}
break;
}
case client_packet_type::MISSING_CHUNKS: {
auto data = peer->getCtx<net_client>();
if (auto player = findEntity(PLAYER_ENTITY_ID, data->instanceId )) {

107
src/server/world/region/File.cpp
View File

@ -34,17 +34,23 @@ void FileRegion::load() {
file.read(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
file.read(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
file.ignore(1);
std::optional<Voxel> avg;
Flags flags = Flags::ZERO;
file.read(reinterpret_cast<char *>(&flags), 1);
if (flags & Flags::HAS_AVERAGE) {
Voxel v;
file.read(reinterpret_cast<char *>(&v), sizeof(v));
avg = v;
}
// Read size
ushort size = 0;
file.read(reinterpret_cast<char *>(&size), sizeof(size));
if (!(flags & Flags::EMPTY)) {
file.read(reinterpret_cast<char *>(&size), sizeof(size));
}
// Ignore content
if(!index.insert({pos, std::make_pair(size, file.tellg())}).second) {
if(!index.emplace(pos, node{avg, size, file.tellg()}).second) {
LOG_E("Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z);
}
file.ignore(size);
@ -57,16 +63,16 @@ void FileRegion::load() {
assert(index.size() == chunkCount);
}
bool FileRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, data& out) {
bool FileRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, std::vector<char>& out) {
std::unique_lock lock(mutex);
const auto it = index.find(pos);
if (it == index.end())
if (it == index.end() || it->second.size == 0)
return false;
data in;
in.resize(it->second.first);
file.seekg(it->second.second);
std::vector<char> in;
in.resize(it->second.size);
file.seekg(it->second.offset);
file.read(in.data(), in.size());
if (auto err = ctx.decompress(in, out)) {
@ -82,17 +88,20 @@ bool FileRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, da
}
return true;
}
void FileRegion::write(const region_chunk_pos& pos, const zstd::write_ctx& ctx, const std::string_view& in) {
auto buffer = std::make_unique<FileRegion::data>();
if (auto err = ctx.compress(in, *buffer)) {
LOG_E("Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< err.value());
return;
void FileRegion::write(const region_chunk_pos& pos, const zstd::write_ctx& ctx, const std::string_view& in, const std::optional<Voxel>& avg) {
std::unique_ptr<std::vector<char>> buffer = nullptr;
if (!in.empty()) {
buffer = std::make_unique<std::vector<char>>();
if (auto err = ctx.compress(in, *buffer)) {
LOG_E("Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< err.value());
return;
}
}
save({{pos, std::move(buffer)}});
save(std::make_optional(to_save{pos, std::move(buffer), avg}));
}
void FileRegion::save(std::optional<std::pair<region_chunk_pos, std::unique_ptr<FileRegion::data>>> added) {
void FileRegion::save(std::optional<to_save> added) {
std::unique_lock lock(mutex);
const auto tmpPath = path + ".tmp";
@ -108,7 +117,7 @@ void FileRegion::save(std::optional<std::pair<region_chunk_pos, std::unique_ptr<
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
}
data tmp;
std::vector<char> tmp;
for(const auto& chunk: index) {
{ // Write pos
region_chunk_pos pos = chunk.first;
@ -117,42 +126,58 @@ void FileRegion::save(std::optional<std::pair<region_chunk_pos, std::unique_ptr<
tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
}
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
tmpFile.put(0);
//MAYBE: store usefull uchar flags
// Write average if present
Flags flags = Flags::ZERO;
if (chunk.second.average.has_value())
flags = (Flags)(flags | Flags::HAS_AVERAGE);
if (chunk.second.size == 0)
flags = (Flags)(flags | Flags::EMPTY);
tmpFile.write(reinterpret_cast<const char *>(&flags), 1);
if (flags & Flags::HAS_AVERAGE) {
Voxel v = chunk.second.average.value();
tmpFile.write(reinterpret_cast<char *>(&v), sizeof(v));
}
// Write size
auto size = chunk.second.first;
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
if (!(flags & Flags::EMPTY)) {
// Write size
auto size = chunk.second.size;
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
// Write content
tmp.resize(size);
file.seekg(chunk.second.second);
file.read(tmp.data(), size);
tmpFile.write(tmp.data(), size);
// Write content
tmp.resize(size);
file.seekg(chunk.second.offset);
file.read(tmp.data(), size);
tmpFile.write(tmp.data(), size);
}
}
if(added.has_value()) {
{ // Write pos
region_chunk_pos pos = added.value().first;
region_chunk_pos pos = added.value().pos;
tmpFile.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
tmpFile.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
}
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
tmpFile.put(0);
//MAYBE: store usefull uchar flags
// Write average if present
Flags flags = Flags::ZERO;
if (added.value().average.has_value())
flags = (Flags)(flags | Flags::HAS_AVERAGE);
if (added.value().data == nullptr)
flags = (Flags)(flags | Flags::EMPTY);
tmpFile.write(reinterpret_cast<const char *>(&flags), 1);
if (flags & Flags::HAS_AVERAGE) {
Voxel v = added.value().average.value();
tmpFile.write(reinterpret_cast<char *>(&v), sizeof(v));
}
// Write size
auto size = added.value().second->size();
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
if (!(flags & Flags::EMPTY)) {
// Write size
auto size = added.value().data->size();
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
// Write content
tmpFile.write(added.value().second->data(), size);
// Write content
tmpFile.write(added.value().data->data(), size);
}
}
if (!tmpFile.good()) {

43
src/server/world/region/File.hpp
View File

@ -4,6 +4,8 @@
#include <shared_mutex>
#include <fstream>
#include "../../../core/world/forward.h"
#include "../../../core/world/Voxel.hpp"
#include "../../../core/data/mem.hpp"
#include "../../../core/utils/zctx.hpp"
#include "../../../core/data/math.hpp"
@ -14,20 +16,49 @@ namespace world::server {
FileRegion(const std::string& folderPath, const area_<region_pos> &pos);
~FileRegion();
typedef std::vector<char> data;
bool read(const region_chunk_pos &pos, const zstd::read_ctx& ctx, data &out);
void write(const region_chunk_pos &pos, const zstd::write_ctx& ctx, const std::string_view &in);
template<typename D>
void averages(D &out) {
std::shared_lock lock(mutex);
out.resize(index.size() * (sizeof(region_chunk_pos) + sizeof(Voxel)));
data::in_view in((uint8_t*)out.data(), out.size());
for(const auto& r: index) {
if (r.second.average.has_value()) {
in.write((const uint8_t*)&r.first, sizeof(r.first));
in.write((const uint8_t*)&r.second.average.value(), sizeof(r.second.average.value()));
}
}
out.resize(in.cur);
}
bool read(const region_chunk_pos &pos, const zstd::read_ctx& ctx, std::vector<char> &out);
void write(const region_chunk_pos &pos, const zstd::write_ctx& ctx, const std::string_view &in, const std::optional<Voxel>& average);
private:
void save(std::optional<std::pair<region_chunk_pos, std::unique_ptr<FileRegion::data>>> added);
struct to_save {
region_chunk_pos pos;
std::unique_ptr<std::vector<char>> data;
std::optional<Voxel> average;
};
void save(std::optional<to_save> added);
std::string path;
//TODO: use tickets to remove unused regions
enum Flags: unsigned char {
ZERO = 0,
HAS_AVERAGE = 1 << 0,
EMPTY = 1 << 1
};
std::shared_mutex mutex;
std::ifstream file;
robin_hood::unordered_map<region_chunk_pos, std::pair<ushort, std::streampos>> index;
struct node {
node(const std::optional<Voxel>& a, ushort s, std::streampos o):
average(a), size(s), offset(o) { }
std::optional<Voxel> average;
ushort size;
std::streampos offset;
};
robin_hood::unordered_map<region_chunk_pos, node> index;
void load();
};

97
src/server/world/region/Memory.cpp
View File

@ -14,13 +14,6 @@ MemoryRegion::MemoryRegion(const std::string &folderPath, const area_<region_pos
MemoryRegion::~MemoryRegion() {
if(!content.empty())
save(changed);
std::unique_lock lock(mutex);
auto it = content.begin();
while(it != content.end()) {
delete it->second;
it = content.erase(it);
}
}
void MemoryRegion::load() {
@ -43,20 +36,30 @@ void MemoryRegion::load() {
file.read(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
file.read(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
file.ignore(1);
// Read average if present
std::optional<Voxel> avg;
Flags flags = Flags::ZERO;
file.read(reinterpret_cast<char *>(&flags), 1);
if (flags & Flags::HAS_AVERAGE) {
Voxel v;
file.read(reinterpret_cast<char *>(&v), sizeof(v));
avg = v;
}
// Read size
ushort size = 0;
file.read(reinterpret_cast<char *>(&size), sizeof(size));
if (!(flags & Flags::EMPTY)) {
file.read(reinterpret_cast<char *>(&size), sizeof(size));
}
// Read content
const auto data = new MemoryRegion::data();
data->resize(size);
file.read(data->data(), data->size());
if(!content.insert({pos, data}).second) {
std::unique_ptr<std::vector<char>> data = nullptr;
if (size > 0) {
data = std::make_unique<std::vector<char>>();
data->resize(size);
file.read(data->data(), data->size());
}
if(!content.emplace(pos, node(avg, std::move(data))).second) {
LOG_E("Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z);
}
file.peek();
@ -68,14 +71,14 @@ void MemoryRegion::load() {
assert(content.size() == chunkCount);
file.close();
}
bool MemoryRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, data& out) {
bool MemoryRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, std::vector<char>& out) {
std::shared_lock lock(mutex);
const auto it = content.find(pos);
if (it == content.end())
if (it == content.end() || it->second.data == nullptr)
return false;
if(auto err = ctx.decompress(*it->second, out)) {
if(auto err = ctx.decompress(*it->second.data.get(), out)) {
LOG_E("Corrupted region chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< err.value());
#ifdef REMOVE_CORRUPTED
@ -91,26 +94,28 @@ bool MemoryRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx,
}
return true;
}
void MemoryRegion::write(const region_chunk_pos& pos, const zstd::write_ctx& ctx, const std::string_view& in) {
const auto buffer = new MemoryRegion::data();
if (auto err = ctx.compress(in, *buffer)) {
LOG_E("Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< err.value());
return;
void MemoryRegion::write(const region_chunk_pos& pos, const zstd::write_ctx& ctx, const std::string_view& in, const std::optional<Voxel>& avg) {
std::unique_ptr<std::vector<char>> buffer = nullptr;
if (!in.empty()) {
buffer = std::make_unique<std::vector<char>>();
if (auto err = ctx.compress(in, *buffer.get())) {
LOG_E("Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< err.value());
return;
}
}
{
std::unique_lock lock(mutex);
// Save buffer
const auto it = content.find(pos);
if (it != content.end())
{
delete it->second;
if (it != content.end()) {
if (buffer == nullptr) {
buffer = std::move(it->second.data);
}
content.erase(it);
}
content.insert({pos, buffer});
content.emplace(pos, node(avg, std::move(buffer)));
changed = true;
}
save(false);
@ -134,10 +139,6 @@ void MemoryRegion::save(bool force) {
}
for(const auto& chunk: content) {
assert(chunk.second->size() < USHRT_MAX);
auto size = (ushort)chunk.second->size();
const auto out = chunk.second->data();
{ // Write pos
region_chunk_pos pos = chunk.first;
file.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
@ -145,17 +146,29 @@ void MemoryRegion::save(bool force) {
file.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
}
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
file.put(0);
//MAYBE: store usefull uchar flags
// Write average if present
Flags flags = Flags::ZERO;
if (chunk.second.average.has_value())
flags = (Flags)(flags | Flags::HAS_AVERAGE);
if (chunk.second.data == nullptr)
flags = (Flags)(flags | Flags::EMPTY);
file.write(reinterpret_cast<const char *>(&flags), sizeof(flags));
if (flags & Flags::HAS_AVERAGE) {
Voxel v = chunk.second.average.value();
file.write(reinterpret_cast<char *>(&v), sizeof(v));
}
// Write size
file.write(reinterpret_cast<char *>(&size), sizeof(size));
if (!(flags & Flags::EMPTY)) {
assert(chunk.second.data->size() < USHRT_MAX);
auto size = (ushort)chunk.second.data->size();
const auto out = chunk.second.data->data();
// Write content
file.write(out, size);
// Write size
file.write(reinterpret_cast<char *>(&size), sizeof(size));
// Write content
file.write(out, size);
}
}
if (!file.good()) {

35
src/server/world/region/Memory.hpp
View File

@ -4,6 +4,8 @@
#include <shared_mutex>
#include <fstream>
#include "../../../core/world/forward.h"
#include "../../../core/world/Voxel.hpp"
#include "../../../core/data/mem.hpp"
#include "../../../core/utils/zctx.hpp"
#include "../../../core/data/math.hpp"
@ -14,10 +16,21 @@ namespace world::server {
MemoryRegion(const std::string& folderPath, const area_<region_pos> &pos);
~MemoryRegion();
typedef std::vector<char> data;
bool read(const region_chunk_pos &pos, const zstd::read_ctx& ctx, data &out);
void write(const region_chunk_pos &pos, const zstd::write_ctx& ctx, const std::string_view &in);
template<typename D>
void averages(D &out) {
std::shared_lock lock(mutex);
out.resize(content.size() * (sizeof(region_chunk_pos) + sizeof(Voxel)));
data::in_view in((uint8_t*)out.data(), out.size());
for(const auto& r: content) {
if (r.second.average.has_value()) {
in.write((const uint8_t*)&r.first, sizeof(r.first));
in.write((const uint8_t*)&r.second.average.value(), sizeof(r.second.average.value()));
}
}
out.resize(in.cur);
}
bool read(const region_chunk_pos &pos, const zstd::read_ctx &ctx, std::vector<char> &out);
void write(const region_chunk_pos &pos, const zstd::write_ctx& ctx, const std::string_view &in, const std::optional<Voxel>& average);
private:
void save(bool force = true);
@ -25,8 +38,20 @@ namespace world::server {
std::string path;
//TODO: use tickets to remove unused regions
enum Flags: unsigned char {
ZERO = 0,
HAS_AVERAGE = 1 << 0,
EMPTY = 1 << 1
};
std::shared_mutex mutex;
robin_hood::unordered_map<region_chunk_pos, data*> content;
struct node {
node(const std::optional<Voxel>& a, std::unique_ptr<std::vector<char>> d):
average(a), data(std::move(d)) { }
std::optional<Voxel> average;
std::unique_ptr<std::vector<char>> data;
};
robin_hood::unordered_map<region_chunk_pos, node> content;
bool changed = false;
void load();