Low memory, lazy and unload regions
This commit is contained in:
parent
752cd4b1a3
commit
d53299ebc8
7
TODO.md
7
TODO.md
|
|
@ -8,12 +8,13 @@
|
|||
- [ ] Remove density
|
||||
- [x] Serialize
|
||||
- [ ] Variable length encoding
|
||||
- [~] Group files
|
||||
- [x] Group files
|
||||
- [x] Zstd + custom grouping
|
||||
- [~] Find best region size
|
||||
- [x] Zstd Train dictionary
|
||||
- [~] Low memory: Keep only ifstream
|
||||
- [ ] Unload unused
|
||||
- [x] Low memory: Keep only ifstream
|
||||
- [x] High memory: Save multiple
|
||||
- [x] Unload unused
|
||||
- [ ] In memory RLE
|
||||
- [x] Edition
|
||||
- [ ] Entity
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ UI::Actions UI::draw(options &options, state &state, const reports &reports, GLu
|
|||
ImGui::PlotHistogram("Count", reports.world.chunk_count.buffer.get(), reports.world.chunk_count.size, 0, std::to_string(reports.world.chunk_count.current()).c_str(), 0);
|
||||
ImGui::PlotHistogram("Loading", reports.world.chunk_load.buffer.get(), reports.world.chunk_load.size, 0, std::to_string(reports.world.chunk_load.current()).c_str(), 0);
|
||||
ImGui::PlotHistogram("Saving", reports.world.chunk_unload.buffer.get(), reports.world.chunk_unload.size, 0, std::to_string(reports.world.chunk_unload.current()).c_str(), 0);
|
||||
ImGui::PlotHistogram("Regions", reports.world.region_count.buffer.get(), reports.world.region_count.size, 0, std::to_string(reports.world.region_count.current()).c_str(), 0);
|
||||
ImGui::Separator();
|
||||
ImGui::Text("Path: %s", options.world.folderPath.c_str());
|
||||
if (ImGui::SliderInt("Load distance", &options.world.loadDistance, 1, options.world.keepDistance) |
|
||||
|
|
|
|||
|
|
@ -1,208 +0,0 @@
|
|||
#include "Region.hpp"
|
||||
|
||||
#include <iostream>
|
||||
#include <filesystem>
|
||||
|
||||
using namespace world;
|
||||
|
||||
Region::Region(const std::string &folderPath, const region_pos &pos) {
|
||||
path = folderPath + '/' + std::to_string(pos.x) + '.' +
|
||||
std::to_string(pos.y) + '.' + std::to_string(pos.z) + ".map";
|
||||
|
||||
load();
|
||||
}
|
||||
Region::~Region() {
|
||||
#ifndef LOW_MEMORY
|
||||
std::unique_lock lock(mutex);
|
||||
auto it = content.begin();
|
||||
while(it != content.end()) {
|
||||
delete it->second;
|
||||
it = content.erase(it);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void Region::load() {
|
||||
std::unique_lock lock(mutex);
|
||||
#ifndef LOW_MEMORY
|
||||
std::ifstream file;
|
||||
#endif
|
||||
file.open(path, std::ios::in | std::ios::binary);
|
||||
if(!file.good()) {
|
||||
return;
|
||||
}
|
||||
// Read header
|
||||
ushort chunkCount; //NOTE: pretty useless
|
||||
file.read(reinterpret_cast<char *>(&chunkCount), sizeof(chunkCount));
|
||||
|
||||
while (!file.eof()) {
|
||||
// Read pos
|
||||
region_chunk_pos pos;
|
||||
file.read(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
|
||||
file.read(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
|
||||
file.read(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
|
||||
|
||||
//NOTE: align uchar pos
|
||||
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
|
||||
file.ignore(1);
|
||||
}
|
||||
|
||||
// Read size
|
||||
ushort size = 0;
|
||||
file.read(reinterpret_cast<char *>(&size), sizeof(size));
|
||||
|
||||
#ifdef LOW_MEMORY
|
||||
// Ignore content
|
||||
const auto saved = index.insert({pos, std::make_pair(size, file.tellg())}).second;
|
||||
file.ignore(size);
|
||||
#else
|
||||
// Read content
|
||||
const auto data = new Region::data();
|
||||
data->resize(size);
|
||||
file.read(data->data(), data->size());
|
||||
const auto saved = content.insert({pos, data}).second;
|
||||
#endif
|
||||
if(!saved) {
|
||||
std::cout << "Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << std::endl;
|
||||
}
|
||||
file.peek();
|
||||
}
|
||||
|
||||
if(file.bad()) {
|
||||
std::cout << "region corrupted read " << path << std::endl;
|
||||
}
|
||||
#ifdef LOW_MEMORY
|
||||
assert(index.size() == chunkCount);
|
||||
#else
|
||||
assert(content.size() == chunkCount);
|
||||
file.close();
|
||||
#endif
|
||||
}
|
||||
bool Region::read(const region_chunk_pos& pos, ZSTD_DCtx* ctx, ZSTD_DDict *dict, data& out) {
|
||||
#ifdef LOW_MEMORY
|
||||
std::unique_lock lock(mutex);
|
||||
|
||||
const auto it = index.find(pos);
|
||||
if (it == index.end())
|
||||
return false;
|
||||
|
||||
auto in = std::make_unique<data>();
|
||||
in->resize(it->second.first);
|
||||
file.seekg(it->second.second);
|
||||
file.read(in->data(), in->size());
|
||||
#else
|
||||
std::shared_lock lock(mutex);
|
||||
|
||||
const auto it = content.find(pos);
|
||||
if (it == content.end())
|
||||
return false;
|
||||
|
||||
auto &in = it->second;
|
||||
#endif
|
||||
|
||||
const auto maxSize = ZSTD_getFrameContentSize(in->data(), in->size());
|
||||
out.resize(maxSize);
|
||||
const auto actualSize = ZSTD_decompress_usingDDict(ctx, out.data(), out.size(), in->data(), in->size(), dict);
|
||||
if(ZSTD_isError(actualSize)) {
|
||||
std::cout << "Corrupted region chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
|
||||
<< ZSTD_getErrorName(actualSize) << std::endl;
|
||||
return false;
|
||||
}
|
||||
out.resize(actualSize);
|
||||
return true;
|
||||
}
|
||||
void Region::save(const region_chunk_pos& pos, ZSTD_CCtx* ctx, ZSTD_CDict* dict, const std::string_view& in) {
|
||||
const auto maxSize = ZSTD_compressBound(in.size());
|
||||
const auto buffer = new Region::data();
|
||||
buffer->resize(maxSize);
|
||||
|
||||
const auto actualSize = ZSTD_compress_usingCDict(ctx, buffer->data(), buffer->capacity(), in.data(), in.size(), dict);
|
||||
if (ZSTD_isError(actualSize)) {
|
||||
std::cout << "Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
|
||||
<< ZSTD_getErrorName(actualSize) << std::endl;
|
||||
return;
|
||||
}
|
||||
buffer->resize(actualSize);
|
||||
|
||||
{
|
||||
std::unique_lock lock(mutex);
|
||||
|
||||
#ifndef LOW_MEMORY
|
||||
{ // Save buffer
|
||||
const auto it = content.find(pos);
|
||||
if(it != content.end()) {
|
||||
delete it->second;
|
||||
content.erase(it);
|
||||
}
|
||||
content.insert({pos, buffer});
|
||||
}
|
||||
const auto& tmpPath = path;
|
||||
#else
|
||||
const auto tmpPath = path + ".tmp";
|
||||
#endif
|
||||
|
||||
std::ofstream tmpFile(tmpPath, std::ios::out | std::ios::binary);
|
||||
if (!tmpFile.good()) {
|
||||
std::cout << "Corrupted region path: " << tmpPath << std::endl;
|
||||
return;
|
||||
}
|
||||
// Write header
|
||||
{
|
||||
#ifdef LOW_MEMORY
|
||||
ushort size = index.size() + 1;
|
||||
#else
|
||||
ushort size = (ushort)content.size();
|
||||
#endif
|
||||
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
|
||||
}
|
||||
|
||||
#ifdef LOW_MEMORY
|
||||
for(const auto& chunk: index) {
|
||||
auto size = chunk.second.first;
|
||||
#else
|
||||
for(const auto& chunk: content) {
|
||||
assert(chunk.second->size() < USHRT_MAX);
|
||||
auto size = (ushort)chunk.second->size();
|
||||
const auto out = chunk.second->data();
|
||||
#endif
|
||||
// Write pos
|
||||
region_chunk_pos pos = chunk.first;
|
||||
tmpFile.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
|
||||
tmpFile.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
|
||||
tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
|
||||
|
||||
//NOTE: align uchar pos
|
||||
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
|
||||
tmpFile.put(0);
|
||||
//MAYBE: store usefull uchar flags
|
||||
}
|
||||
|
||||
// Write size
|
||||
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
|
||||
|
||||
// Write content
|
||||
#ifdef LOW_MEMORY
|
||||
data tmp;
|
||||
tmp.resize(size);
|
||||
file.seekg(chunk.second.second);
|
||||
const auto out = tmp.data();
|
||||
file.read(out, size);
|
||||
#endif
|
||||
tmpFile.write(out, size);
|
||||
}
|
||||
|
||||
if (!tmpFile.good()) {
|
||||
std::cout << "region corrupted write " << tmpPath << std::endl;
|
||||
tmpFile.close();
|
||||
return;
|
||||
}
|
||||
tmpFile.close();
|
||||
|
||||
#ifdef LOW_MEMORY
|
||||
index.clear();
|
||||
file.close();
|
||||
std::filesystem::rename(tmpPath, path);
|
||||
load();
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <shared_mutex>
|
||||
#include <fstream>
|
||||
#include <zstd.h>
|
||||
#include "forward.h"
|
||||
#include "../data/math.hpp"
|
||||
|
||||
namespace world {
|
||||
///Group of chunks saved as a single file
|
||||
class Region {
|
||||
public:
|
||||
Region(const std::string& folderPath, const region_pos &pos);
|
||||
~Region();
|
||||
|
||||
typedef std::vector<char> data;
|
||||
|
||||
bool read(const region_chunk_pos &pos, ZSTD_DCtx *dctx, ZSTD_DDict *ddict, data &out);
|
||||
void save(const region_chunk_pos &pos, ZSTD_CCtx *cctx, ZSTD_CDict *cdict, const std::string_view &in);
|
||||
|
||||
private:
|
||||
std::string path;
|
||||
//TODO: try dictionnary
|
||||
//TODO: use tickets to remove unused regions
|
||||
|
||||
std::shared_mutex mutex;
|
||||
#ifdef LOW_MEMORY
|
||||
std::ifstream file;
|
||||
robin_hood::unordered_flat_map<region_chunk_pos, std::pair<ushort, std::streampos>> index;
|
||||
#else
|
||||
robin_hood::unordered_flat_map<region_chunk_pos, data*> content;
|
||||
#endif
|
||||
|
||||
void load();
|
||||
};
|
||||
}
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
using namespace world;
|
||||
|
||||
Universe::Universe(const Universe::options &options): generator(42), contouring(std::make_shared<contouring::Dummy>()) {
|
||||
Universe::Universe(const Universe::options &options): generator(42), regionDict("content/zstd.dict"), contouring(std::make_shared<contouring::Dummy>()) {
|
||||
setOptions(options);
|
||||
folderPath = options.folderPath;
|
||||
struct vec_istream: std::streambuf {
|
||||
|
|
@ -19,69 +19,52 @@ Universe::Universe(const Universe::options &options): generator(42), contouring(
|
|||
running = true;
|
||||
std::filesystem::create_directories(folderPath);
|
||||
|
||||
{ // Load dictionary
|
||||
std::ifstream is("content/zstd.dict", std::ios::in | std::ios::binary | std::ios::ate);
|
||||
if(!is.good()) {
|
||||
std::cout << "missing dict" << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
const auto end = is.tellg();
|
||||
is.seekg(0, std::ios::beg);
|
||||
std::vector<char> dict(end - is.tellg());
|
||||
is.read(dict.data(), dict.size());
|
||||
is.close();
|
||||
cdict = ZSTD_createCDict(dict.data(), dict.size(), ZSTD_CLEVEL_DEFAULT);
|
||||
ddict = ZSTD_createDDict(dict.data(), dict.size());
|
||||
}
|
||||
|
||||
// Load workers
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
loadWorkers.emplace_back([&] {
|
||||
ZSTD_DCtx *dctx = ZSTD_createDCtx();
|
||||
const auto ctx = regionDict.make_reader();
|
||||
while (running) {
|
||||
chunk_pos ctx;
|
||||
chunk_pos pos;
|
||||
loadQueue.wait();
|
||||
if (loadQueue.pop(ctx)) {
|
||||
if (loadQueue.pop(pos)) {
|
||||
//MAYBE: loadQueue.take to avoid duplicated work on fast move
|
||||
rmt_ScopedCPUSample(ProcessLoad, 0);
|
||||
const region_pos rPos = glm::divide(ctx, region_chunk_pos(REGION_LENGTH));
|
||||
const region_chunk_pos cPos = glm::modulo(ctx, region_chunk_pos(REGION_LENGTH));
|
||||
const region_pos rPos = glm::divide(pos, region_chunk_pos(REGION_LENGTH));
|
||||
const region_chunk_pos cPos = glm::modulo(pos, region_chunk_pos(REGION_LENGTH));
|
||||
const auto reg = getRegion(rPos);
|
||||
Region::data data;
|
||||
if(reg->read(cPos, dctx, ddict, data)) {
|
||||
if(reg->read(cPos, ctx, data)) {
|
||||
rmt_ScopedCPUSample(ProcessRead, 0);
|
||||
vec_istream idata(data);
|
||||
std::istream iss(&idata);
|
||||
loadedQueue.push({ctx, std::make_shared<Chunk>(iss)});
|
||||
loadedQueue.push({pos, std::make_shared<Chunk>(iss)});
|
||||
} else {
|
||||
rmt_ScopedCPUSample(ProcessGenerate, 0);
|
||||
loadedQueue.push({ctx, std::make_shared<Chunk>(ctx, generator)});
|
||||
loadedQueue.push({pos, std::make_shared<Chunk>(pos, generator)});
|
||||
}
|
||||
}
|
||||
}
|
||||
ZSTD_freeDCtx(dctx);
|
||||
});
|
||||
}
|
||||
|
||||
// Save workers
|
||||
for (size_t i = 0; i < 2; i++) {
|
||||
saveWorkers.emplace_back([&] {
|
||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||
const auto ctx = regionDict.make_writer();
|
||||
while (running) {
|
||||
robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> ctx;
|
||||
robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> task;
|
||||
saveQueue.wait();
|
||||
if (saveQueue.pop(ctx) && ctx.second->isModified()) {
|
||||
if (saveQueue.pop(task) && task.second->isModified()) {
|
||||
//MAYBE: queue.take to avoid concurent write or duplicated work on fast move
|
||||
rmt_ScopedCPUSample(ProcessSave, 0);
|
||||
std::ostringstream out;
|
||||
ctx.second->write(out);
|
||||
const region_pos rPos = glm::divide(ctx.first, region_chunk_pos(REGION_LENGTH));
|
||||
const region_chunk_pos cPos = glm::modulo(ctx.first, region_chunk_pos(REGION_LENGTH));
|
||||
task.second->write(out);
|
||||
const region_pos rPos = glm::divide(task.first, region_chunk_pos(REGION_LENGTH));
|
||||
const region_chunk_pos cPos = glm::modulo(task.first, region_chunk_pos(REGION_LENGTH));
|
||||
const auto reg = getRegion(rPos);
|
||||
reg->save(cPos, cctx, cdict, out.str());
|
||||
reg->write(cPos, ctx, out.str());
|
||||
}
|
||||
}
|
||||
ZSTD_freeCCtx(cctx);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -92,11 +75,14 @@ Universe::~Universe() {
|
|||
for(auto& pair: chunks) {
|
||||
saveQueue.push(pair);
|
||||
}
|
||||
if(saveQueue.size() > 0) {
|
||||
std::cout << "Saving..." << std::endl;
|
||||
}
|
||||
while(saveQueue.size() > 0) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
if (auto size = saveQueue.size(); size > 0) {
|
||||
std::cout << std::endl;
|
||||
do {
|
||||
std::cout << "\rSaving... " << size << " " << std::flush;
|
||||
std::this_thread::sleep_for(std::chrono::microseconds(500));
|
||||
size = saveQueue.size();
|
||||
} while (size > 0);
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
running = false;
|
||||
|
|
@ -111,9 +97,6 @@ Universe::~Universe() {
|
|||
if (worker.joinable())
|
||||
worker.join();
|
||||
}
|
||||
|
||||
ZSTD_freeCDict(cdict);
|
||||
ZSTD_freeDDict(ddict);
|
||||
}
|
||||
|
||||
std::shared_ptr<Region> Universe::getRegion(const region_pos& pos) {
|
||||
|
|
@ -188,6 +171,20 @@ void Universe::update(const camera_pos& pos, Universe::report& rep) {
|
|||
}
|
||||
}
|
||||
rep.chunk_count.push(chunks.size());
|
||||
|
||||
{
|
||||
rmt_ScopedCPUSample(Region, 0);
|
||||
std::unique_lock lock(regionMutex);
|
||||
const auto me = glm::divide(last_pos, glm::ivec3(REGION_LENGTH));
|
||||
for (auto it = regionCache.begin(); it != regionCache.end();) {
|
||||
if (glm::length2(it->first - me) > keepDistance) {
|
||||
std::cout << "rem" << std::endl;
|
||||
it = regionCache.erase(it);
|
||||
} else {
|
||||
it++;
|
||||
}
|
||||
}
|
||||
}
|
||||
rep.region_count.push(regionCache.size());
|
||||
}
|
||||
void Universe::setOptions(const Universe::options& options) {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
#include "../data/geometry/Ray.hpp"
|
||||
#include "forward.h"
|
||||
#include "Voxel.hpp"
|
||||
#include "Region.hpp"
|
||||
#include "region/index.hpp"
|
||||
#include "Generator.hpp"
|
||||
typedef glm::vec3 camera_pos;
|
||||
|
||||
|
|
@ -100,9 +100,7 @@ namespace world {
|
|||
|
||||
std::shared_mutex regionMutex; //MAYBE: shared_guard
|
||||
robin_hood::unordered_map<region_pos, std::shared_ptr<Region>> regionCache;
|
||||
ZSTD_CDict *cdict;
|
||||
ZSTD_DDict *ddict;
|
||||
//TODO: region pick list
|
||||
dict_set regionDict;
|
||||
std::shared_ptr<Region> getRegion(const region_pos &);
|
||||
|
||||
/// Contouring worker
|
||||
|
|
|
|||
|
|
@ -5,6 +5,5 @@
|
|||
|
||||
namespace world {
|
||||
class Chunk;
|
||||
class Region;
|
||||
typedef robin_hood::unordered_map<chunk_pos, std::shared_ptr<Chunk>> chunk_map;
|
||||
}
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
#include "File.hpp"
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
using namespace world;
|
||||
|
||||
#define REMOVE_CORRUPTED 1
|
||||
|
||||
FileRegion::FileRegion(const std::string &folderPath, const region_pos &pos) {
|
||||
path = folderPath + '/' + std::to_string(pos.x) + '.' +
|
||||
std::to_string(pos.y) + '.' + std::to_string(pos.z) + ".map";
|
||||
|
||||
load();
|
||||
}
|
||||
FileRegion::~FileRegion() {
|
||||
file.close();
|
||||
}
|
||||
|
||||
void FileRegion::load() {
|
||||
std::unique_lock lock(mutex);
|
||||
|
||||
file.open(path, std::ios::in | std::ios::binary);
|
||||
if(!file.good()) {
|
||||
return;
|
||||
}
|
||||
// Read header
|
||||
ushort chunkCount; //NOTE: pretty useless
|
||||
file.read(reinterpret_cast<char *>(&chunkCount), sizeof(chunkCount));
|
||||
|
||||
while (!file.eof()) {
|
||||
// Read pos
|
||||
region_chunk_pos pos;
|
||||
file.read(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
|
||||
file.read(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
|
||||
file.read(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
|
||||
|
||||
//NOTE: align uchar pos
|
||||
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
|
||||
file.ignore(1);
|
||||
}
|
||||
|
||||
// Read size
|
||||
ushort size = 0;
|
||||
file.read(reinterpret_cast<char *>(&size), sizeof(size));
|
||||
|
||||
// Ignore content
|
||||
if(!index.insert({pos, std::make_pair(size, file.tellg())}).second) {
|
||||
std::cout << "Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << std::endl;
|
||||
}
|
||||
file.ignore(size);
|
||||
file.peek();
|
||||
}
|
||||
|
||||
if(file.bad()) {
|
||||
std::cout << "region corrupted read " << path << std::endl;
|
||||
}
|
||||
|
||||
assert(index.size() == chunkCount);
|
||||
}
|
||||
bool FileRegion::read(const region_chunk_pos& pos, const read_ctx& ctx, data& out) {
|
||||
std::unique_lock lock(mutex);
|
||||
|
||||
const auto it = index.find(pos);
|
||||
if (it == index.end())
|
||||
return false;
|
||||
|
||||
auto in = std::make_unique<data>();
|
||||
in->resize(it->second.first);
|
||||
file.seekg(it->second.second);
|
||||
file.read(in->data(), in->size());
|
||||
|
||||
const auto maxSize = ZSTD_getFrameContentSize(in->data(), in->size());
|
||||
out.resize(maxSize);
|
||||
const auto actualSize = ZSTD_decompress_usingDDict(ctx.ctx, out.data(), out.size(), in->data(), in->size(), ctx.dict);
|
||||
if(ZSTD_isError(actualSize)) {
|
||||
std::cout << "Corrupted region chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
|
||||
<< ZSTD_getErrorName(actualSize) << std::endl;
|
||||
#ifdef REMOVE_CORRUPTED
|
||||
std::cout << "Removing" << std::endl;
|
||||
index.erase(it);
|
||||
lock.unlock();
|
||||
save(std::nullopt);
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
out.resize(actualSize);
|
||||
return true;
|
||||
}
|
||||
void FileRegion::write(const region_chunk_pos& pos, const write_ctx& ctx, const std::string_view& in) {
|
||||
const auto maxSize = ZSTD_compressBound(in.size());
|
||||
auto buffer = std::make_unique<FileRegion::data>();
|
||||
buffer->resize(maxSize);
|
||||
|
||||
const auto actualSize = ZSTD_compress_usingCDict(ctx.ctx, buffer->data(), buffer->capacity(), in.data(), in.size(), ctx.dict);
|
||||
if (ZSTD_isError(actualSize)) {
|
||||
std::cout << "Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
|
||||
<< ZSTD_getErrorName(actualSize) << std::endl;
|
||||
return;
|
||||
}
|
||||
buffer->resize(actualSize);
|
||||
save({{pos, std::move(buffer)}});
|
||||
}
|
||||
|
||||
void FileRegion::save(std::optional<std::pair<region_chunk_pos, std::unique_ptr<FileRegion::data>>> added) {
|
||||
std::unique_lock lock(mutex);
|
||||
|
||||
const auto tmpPath = path + ".tmp";
|
||||
|
||||
std::ofstream tmpFile(tmpPath, std::ios::out | std::ios::binary);
|
||||
if (!tmpFile.good()) {
|
||||
std::cout << "Corrupted region path: " << tmpPath << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
{ // Write header
|
||||
ushort size = index.size() + (added.has_value() ? 1 : 0);
|
||||
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
|
||||
}
|
||||
|
||||
data tmp;
|
||||
for(const auto& chunk: index) {
|
||||
{ // Write pos
|
||||
region_chunk_pos pos = chunk.first;
|
||||
tmpFile.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
|
||||
tmpFile.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
|
||||
tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
|
||||
}
|
||||
|
||||
//NOTE: align uchar pos
|
||||
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
|
||||
tmpFile.put(0);
|
||||
//MAYBE: store usefull uchar flags
|
||||
}
|
||||
|
||||
// Write size
|
||||
auto size = chunk.second.first;
|
||||
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
|
||||
|
||||
// Write content
|
||||
tmp.resize(size);
|
||||
file.seekg(chunk.second.second);
|
||||
file.read(tmp.data(), size);
|
||||
tmpFile.write(tmp.data(), size);
|
||||
}
|
||||
if(added.has_value()) {
|
||||
{ // Write pos
|
||||
region_chunk_pos pos = added.value().first;
|
||||
tmpFile.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
|
||||
tmpFile.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
|
||||
tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
|
||||
}
|
||||
|
||||
//NOTE: align uchar pos
|
||||
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
|
||||
tmpFile.put(0);
|
||||
//MAYBE: store usefull uchar flags
|
||||
}
|
||||
|
||||
// Write size
|
||||
auto size = added.value().second->size();
|
||||
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
|
||||
|
||||
// Write content
|
||||
tmpFile.write(added.value().second->data(), size);
|
||||
}
|
||||
|
||||
if (!tmpFile.good()) {
|
||||
std::cout << "region corrupted write " << tmpPath << std::endl;
|
||||
tmpFile.close();
|
||||
return;
|
||||
}
|
||||
tmpFile.close();
|
||||
|
||||
index.clear();
|
||||
file.close();
|
||||
std::filesystem::rename(tmpPath, path);
|
||||
load();
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <shared_mutex>
|
||||
#include "../forward.h"
|
||||
#include "common.hpp"
|
||||
#include "../../data/math.hpp"
|
||||
|
||||
namespace world {
|
||||
///Group of chunks saved as a single file only pointer
|
||||
class FileRegion {
|
||||
public:
|
||||
FileRegion(const std::string& folderPath, const region_pos &pos);
|
||||
~FileRegion();
|
||||
|
||||
typedef std::vector<char> data;
|
||||
|
||||
bool read(const region_chunk_pos &pos, const read_ctx& ctx, data &out);
|
||||
void write(const region_chunk_pos &pos, const write_ctx& ctx, const std::string_view &in);
|
||||
|
||||
private:
|
||||
void save(std::optional<std::pair<region_chunk_pos, std::unique_ptr<FileRegion::data>>> added);
|
||||
|
||||
std::string path;
|
||||
//TODO: use tickets to remove unused regions
|
||||
|
||||
std::shared_mutex mutex;
|
||||
std::ifstream file;
|
||||
robin_hood::unordered_flat_map<region_chunk_pos, std::pair<ushort, std::streampos>> index;
|
||||
|
||||
void load();
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
#include "Memory.hpp"
|
||||
|
||||
using namespace world;
|
||||
|
||||
#define REMOVE_CORRUPTED 1
|
||||
#define LAZYNESS 8
|
||||
|
||||
MemoryRegion::MemoryRegion(const std::string &folderPath, const region_pos &pos) {
|
||||
path = folderPath + '/' + std::to_string(pos.x) + '.' +
|
||||
std::to_string(pos.y) + '.' + std::to_string(pos.z) + ".map";
|
||||
|
||||
load();
|
||||
}
|
||||
MemoryRegion::~MemoryRegion() {
|
||||
if(!content.empty())
|
||||
save(changed);
|
||||
|
||||
std::unique_lock lock(mutex);
|
||||
auto it = content.begin();
|
||||
while(it != content.end()) {
|
||||
delete it->second;
|
||||
it = content.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
void MemoryRegion::load() {
|
||||
std::unique_lock lock(mutex);
|
||||
|
||||
std::ifstream file;
|
||||
file.open(path, std::ios::in | std::ios::binary);
|
||||
if(!file.good()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Read header
|
||||
ushort chunkCount; //NOTE: pretty useless
|
||||
file.read(reinterpret_cast<char *>(&chunkCount), sizeof(chunkCount));
|
||||
|
||||
while (!file.eof()) {
|
||||
// Read pos
|
||||
region_chunk_pos pos;
|
||||
file.read(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
|
||||
file.read(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
|
||||
file.read(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
|
||||
|
||||
//NOTE: align uchar pos
|
||||
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
|
||||
file.ignore(1);
|
||||
}
|
||||
|
||||
// Read size
|
||||
ushort size = 0;
|
||||
file.read(reinterpret_cast<char *>(&size), sizeof(size));
|
||||
|
||||
// Read content
|
||||
const auto data = new MemoryRegion::data();
|
||||
data->resize(size);
|
||||
file.read(data->data(), data->size());
|
||||
if(!content.insert({pos, data}).second) {
|
||||
std::cout << "Duplicated chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << std::endl;
|
||||
}
|
||||
file.peek();
|
||||
}
|
||||
|
||||
if(file.bad()) {
|
||||
std::cout << "region corrupted read " << path << std::endl;
|
||||
}
|
||||
assert(content.size() == chunkCount);
|
||||
file.close();
|
||||
}
|
||||
bool MemoryRegion::read(const region_chunk_pos& pos, const read_ctx& ctx, data& out) {
|
||||
std::shared_lock lock(mutex);
|
||||
|
||||
const auto it = content.find(pos);
|
||||
if (it == content.end())
|
||||
return false;
|
||||
|
||||
auto &in = it->second;
|
||||
|
||||
const auto maxSize = ZSTD_getFrameContentSize(in->data(), in->size());
|
||||
out.resize(maxSize);
|
||||
const auto actualSize = ZSTD_decompress_usingDDict(ctx.ctx, out.data(), out.size(), in->data(), in->size(), ctx.dict);
|
||||
if(ZSTD_isError(actualSize)) {
|
||||
std::cout << "Corrupted region chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
|
||||
<< ZSTD_getErrorName(actualSize) << std::endl;
|
||||
#ifdef REMOVE_CORRUPTED
|
||||
std::cout << "Removing" << std::endl;
|
||||
lock.unlock();
|
||||
{
|
||||
std::unique_lock ulock(mutex);
|
||||
content.erase(it);
|
||||
}
|
||||
save(true);
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
out.resize(actualSize);
|
||||
return true;
|
||||
}
|
||||
void MemoryRegion::write(const region_chunk_pos& pos, const write_ctx& ctx, const std::string_view& in) {
|
||||
const auto maxSize = ZSTD_compressBound(in.size());
|
||||
const auto buffer = new MemoryRegion::data();
|
||||
buffer->resize(maxSize);
|
||||
|
||||
const auto actualSize = ZSTD_compress_usingCDict(ctx.ctx, buffer->data(), buffer->capacity(), in.data(), in.size(), ctx.dict);
|
||||
if (ZSTD_isError(actualSize)) {
|
||||
std::cout << "Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
|
||||
<< ZSTD_getErrorName(actualSize) << std::endl;
|
||||
return;
|
||||
}
|
||||
buffer->resize(actualSize);
|
||||
|
||||
{
|
||||
std::unique_lock lock(mutex);
|
||||
|
||||
// Save buffer
|
||||
const auto it = content.find(pos);
|
||||
if (it != content.end())
|
||||
{
|
||||
delete it->second;
|
||||
content.erase(it);
|
||||
}
|
||||
content.insert({pos, buffer});
|
||||
changed = true;
|
||||
}
|
||||
save(false);
|
||||
}
|
||||
|
||||
void MemoryRegion::save(bool force) {
|
||||
if(!force && rand() % LAZYNESS == 0)
|
||||
return;
|
||||
|
||||
std::unique_lock lock(mutex);
|
||||
|
||||
std::ofstream file(path, std::ios::out | std::ios::binary);
|
||||
if (!file.good()) {
|
||||
std::cout << "Corrupted region path: " << path << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
{ // Write header
|
||||
ushort size = (ushort)content.size();
|
||||
file.write(reinterpret_cast<char *>(&size), sizeof(size));
|
||||
}
|
||||
|
||||
for(const auto& chunk: content) {
|
||||
assert(chunk.second->size() < USHRT_MAX);
|
||||
auto size = (ushort)chunk.second->size();
|
||||
const auto out = chunk.second->data();
|
||||
|
||||
{ // Write pos
|
||||
region_chunk_pos pos = chunk.first;
|
||||
file.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
|
||||
file.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
|
||||
file.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
|
||||
}
|
||||
|
||||
//NOTE: align uchar pos
|
||||
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
|
||||
file.put(0);
|
||||
//MAYBE: store usefull uchar flags
|
||||
}
|
||||
|
||||
// Write size
|
||||
file.write(reinterpret_cast<char *>(&size), sizeof(size));
|
||||
|
||||
// Write content
|
||||
file.write(out, size);
|
||||
}
|
||||
|
||||
if (!file.good()) {
|
||||
std::cout << "region corrupted write " << path << std::endl;
|
||||
file.close();
|
||||
return;
|
||||
}
|
||||
file.close();
|
||||
changed = false;
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <shared_mutex>
|
||||
#include <zstd.h>
|
||||
#include "../forward.h"
|
||||
#include "common.hpp"
|
||||
#include "../../data/math.hpp"
|
||||
|
||||
namespace world {
|
||||
///Group of chunks saved as a single file in memory
|
||||
class MemoryRegion {
|
||||
public:
|
||||
MemoryRegion(const std::string& folderPath, const region_pos &pos);
|
||||
~MemoryRegion();
|
||||
|
||||
typedef std::vector<char> data;
|
||||
|
||||
bool read(const region_chunk_pos &pos, const read_ctx& ctx, data &out);
|
||||
void write(const region_chunk_pos &pos, const write_ctx& ctx, const std::string_view &in);
|
||||
|
||||
private:
|
||||
void save(bool force = true);
|
||||
|
||||
std::string path;
|
||||
//TODO: use tickets to remove unused regions
|
||||
|
||||
std::shared_mutex mutex;
|
||||
robin_hood::unordered_flat_map<region_chunk_pos, data*> content;
|
||||
bool changed = false;
|
||||
|
||||
void load();
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
#pragma once
|
||||
|
||||
#include <zstd.h>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <vector>
|
||||
#include <cassert>
|
||||
|
||||
namespace world {
|
||||
struct read_ctx {
|
||||
~read_ctx() {
|
||||
ZSTD_freeDCtx(ctx);
|
||||
}
|
||||
ZSTD_DCtx *ctx;
|
||||
ZSTD_DDict *dict;
|
||||
};
|
||||
struct write_ctx {
|
||||
~write_ctx() {
|
||||
ZSTD_freeCCtx(ctx);
|
||||
}
|
||||
ZSTD_CCtx *ctx;
|
||||
ZSTD_CDict *dict;
|
||||
};
|
||||
|
||||
class dict_set {
|
||||
public:
|
||||
dict_set(const std::string& path) {
|
||||
std::ifstream is(path, std::ios::in | std::ios::binary | std::ios::ate);
|
||||
if(!is.good()) {
|
||||
std::cout << "missing dict " << path << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
const auto end = is.tellg();
|
||||
is.seekg(0, std::ios::beg);
|
||||
std::vector<char> dict(end - is.tellg());
|
||||
is.read(dict.data(), dict.size());
|
||||
is.close();
|
||||
c = ZSTD_createCDict(dict.data(), dict.size(), ZSTD_CLEVEL_DEFAULT);
|
||||
assert(c != NULL);
|
||||
d = ZSTD_createDDict(dict.data(), dict.size());
|
||||
assert(d != NULL);
|
||||
}
|
||||
~dict_set() {
|
||||
ZSTD_freeCDict(c);
|
||||
ZSTD_freeDDict(d);
|
||||
}
|
||||
|
||||
read_ctx make_reader() const {
|
||||
return read_ctx{ZSTD_createDCtx(), d};
|
||||
}
|
||||
write_ctx make_writer() const {
|
||||
return write_ctx{ZSTD_createCCtx(), c};
|
||||
}
|
||||
private:
|
||||
ZSTD_CDict *c;
|
||||
ZSTD_DDict *d;
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#ifdef LOW_MEMORY
|
||||
#include "File.hpp"
|
||||
namespace world {typedef FileRegion Region;}
|
||||
#else
|
||||
#include "Memory.hpp"
|
||||
namespace world {typedef MemoryRegion Region;}
|
||||
#endif
|
||||
Loading…
Reference in New Issue