me
/
Univerxel
1
0
Fork 0
Code Releases Activity

Zstd region

This commit is contained in:
May B. 2020-07-30 18:35:13 +02:00
parent 8267793035
commit 8d967cee06
11 changed files with 383 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
CMakeLists.txt
View File

@ -17,12 +17,13 @@ if(CCACHE_FOUND)
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
endif(CCACHE_FOUND)
set(ALL_LIBS
set(LINKED_LIBS
${OPENGL_LIBRARY}
glfw
GLEW
pthread
dl
zstd
)
add_definitions(
@ -37,11 +38,18 @@ add_definitions(
file(GLOB_RECURSE SOURCES "src/*.cpp")
file(GLOB INCLUDE_SOURCES "include/imgui-1.76/*.cpp" "include/FastNoiseSIMD/*.cpp" "include/Remotery/lib/*.c")
set(INCLUDE_LIBS
"include/imgui-1.76"
"include/FastNoiseSIMD"
"include/toml++"
"include/Remotery/lib"
"include/robin_hood"
)
add_executable(univerxel ${SOURCES} ${INCLUDE_SOURCES})
target_compile_features(univerxel PUBLIC cxx_std_17)
target_link_libraries(univerxel ${ALL_LIBS})
target_include_directories(univerxel PRIVATE "include/imgui-1.76/" "include/FastNoiseSIMD/" "include/toml++/" "include/Remotery/lib/" "include/robin_hood")
target_link_libraries(univerxel ${LINKED_LIBS})
target_include_directories(univerxel PRIVATE ${INCLUDE_LIBS})
if(PROFILING)
target_compile_definitions(univerxel PRIVATE RMT_ENABLED=1 RMT_USE_OPENGL=1)
else(PROFILING)

1
README.md
View File

@ -46,6 +46,7 @@ To get a local copy up and running, follow these simple steps.
* OpenGL
* GLFw
* Glew
* Zstd
#### Optionally

10
TODO.md
View File

@ -5,9 +5,15 @@
- [x] Density
- [x] Robin hood map
- [ ] Octree world
- [ ] Remove density
- [x] Serialize
- [ ] Variable length encoding
- [ ] Group files
- [~] Group files
- [x] Zstd + custom grouping
- [~] Find best region size
- [ ] Zstd Train dictionary
- [~] Low memory: Keep only ifstream
- [ ] Unload unused
- [ ] In memory RLE
- [x] Edition
- [ ] Entity
@ -24,6 +30,8 @@
- Xtree-memory
- [ ] Server
- [ ] ZeroMQ
- [ ] Mutex guard
- [ ] Shared mutex
## Rendering
- [x] Render triangle

8
src/data/glm.hpp
View File

@ -6,12 +6,16 @@
namespace glm {
typedef vec<3, long long> lvec3;
typedef vec<4, long long> lvec4;
}
typedef vec<3, ushort> usvec3;
typedef vec<3, unsigned char> ucvec3;
} // namespace glm
typedef glm::vec3 camera_pos;
typedef glm::lvec3 voxel_pos;
typedef glm::ivec3 chunk_pos;
typedef glm::vec<3, ushort> chunk_voxel_pos;
typedef glm::ucvec3 chunk_voxel_pos;
typedef glm::ivec3 region_pos;
typedef glm::ucvec3 region_chunk_pos;
namespace glm {
ivec3 inline iround(const vec3& p) {

20
src/world/Chunk.cpp
View File

@ -7,6 +7,7 @@ using namespace world;
#define DENSITY 0.f
#define GRANULARITY 30.f
#define RLE 1
Chunk::Chunk(const chunk_pos& pos, Generator& rnd) {
const auto [densitySet, materialSet] = rnd.getChunk(pos, CHUNK_LENGTH);
@ -19,7 +20,8 @@ Chunk::Chunk(const chunk_pos& pos, Generator& rnd) {
FastNoiseSIMD::FreeNoiseSet(densitySet);
FastNoiseSIMD::FreeNoiseSet(materialSet);
}
Chunk::Chunk(std::ifstream& str) {
Chunk::Chunk(std::istream& str) {
#ifdef RLE
ushort i = 0;
while(!str.eof()) {
ushort count;
@ -34,11 +36,17 @@ Chunk::Chunk(std::ifstream& str) {
}
}
assert(i == CHUNK_SIZE-1);
#else
for(auto& voxel: voxels) {
str.read(reinterpret_cast<char *>(&voxel.Density), sizeof(Voxel::Density));
str.read(reinterpret_cast<char *>(&voxel.Material), sizeof(Voxel::Material));
}
#endif
}
Chunk::~Chunk() { }
void Chunk::write(std::ofstream& str) const {
//TODO: variable length encoding (short ones)
void Chunk::write(std::ostream& str) const {
#ifdef RLE
auto it = voxels.begin();
ushort counter = 1;
Voxel current = *it;
@ -58,6 +66,12 @@ void Chunk::write(std::ofstream& str) const {
counter++;
}
}
#else
for(auto current: voxels) {
str.write(reinterpret_cast<char *>(&current.Density), sizeof(current.Density));
str.write(reinterpret_cast<char *>(&current.Material), sizeof(current.Material));
}
#endif
}
std::optional<Faces> Chunk::update() {

6
src/world/Chunk.hpp
View File

@ -3,7 +3,7 @@
#include "Generator.hpp"
#include "Voxel.hpp"
#include "../data/geometry/Faces.hpp"
#include <fstream>
#include <sstream>
/// Chunk length
#define CHUNK_LENGTH 32
@ -16,7 +16,7 @@ namespace world {
struct Chunk {
public:
Chunk(const chunk_pos& pos, Generator& rnd);
Chunk(std::ifstream& str);
Chunk(std::istream& str);
~Chunk();
/// Update voxels
@ -54,7 +54,7 @@ namespace world {
inline bool isModified() const { return modified; }
// Write to file.
// Using RLE
void write(std::ofstream& str) const;
void write(std::ostream& str) const;
static inline chunk_voxel_pos getPosition(ushort idx) {
return chunk_voxel_pos(idx / CHUNK_LENGTH2, (idx / CHUNK_LENGTH) % CHUNK_LENGTH, idx % CHUNK_LENGTH);

206
src/world/Region.cpp Normal file
View File

@ -0,0 +1,206 @@
#include "Region.hpp"
#include <iostream>
#include <filesystem>
using namespace world;
Region::Region(const std::string &folderPath, const region_pos &pos) {
path = folderPath + '/' + std::to_string(pos.x) + '.' +
std::to_string(pos.y) + '.' + std::to_string(pos.z) + ".map";
load();
}
Region::~Region() {
#ifndef LOW_MEMORY
std::unique_lock lock(mutex);
auto it = content.begin();
while(it != content.end()) {
delete it->second;
it = content.erase(it);
}
#endif
}
void Region::load() {
std::unique_lock lock(mutex);
#ifndef LOW_MEMORY
std::ifstream file;
#endif
file.open(path, std::ios::in | std::ios::binary);
if(!file.good()) {
return;
}
// Read header
ushort chunkCount; //NOTE: pretty useless
file.read(reinterpret_cast<char *>(&chunkCount), sizeof(chunkCount));
while (!file.eof()) {
// Read pos
region_chunk_pos pos;
file.read(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
file.read(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
file.read(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
file.ignore(1);
}
// Read size
ushort size = 0;
file.read(reinterpret_cast<char *>(&size), sizeof(size));
#ifdef LOW_MEMORY
// Ignore content
const auto saved = index.insert({pos, std::make_pair(size, file.tellg())}).second;
file.ignore(size);
#else
// Read content
const auto data = new Region::data();
data->resize(size);
file.read(data->data(), data->size());
const auto saved = content.insert({pos, data}).second;
#endif
if(!saved) {
std::cout << "Duplicated chunk: " << path << ":" << pos.x << "." << pos.y << "." << pos.z << std::endl;
}
file.peek();
}
if(file.bad()) {
std::cout << "region corrupted read " << path << std::endl;
}
#ifdef LOW_MEMORY
assert(index.size() == chunkCount);
#else
assert(content.size() == chunkCount);
file.close();
#endif
}
bool Region::read(const region_chunk_pos& pos, ZSTD_DCtx* dctx, data& out) {
#ifdef LOW_MEMORY
std::unique_lock lock(mutex);
const auto it = index.find(pos);
if (it == index.end())
return false;
auto in = std::make_unique<data>();
in->resize(it->second.first);
file.seekg(it->second.second);
file.read(in->data(), in->size());
#else
std::shared_lock lock(mutex);
const auto it = content.find(pos);
if (it == content.end())
return false;
auto &in = it->second;
#endif
const auto maxSize = ZSTD_getFrameContentSize(in->data(), in->size());
out.resize(maxSize);
const auto actualSize = ZSTD_decompressDCtx(dctx, out.data(), out.size(), in->data(), in->size());
if(ZSTD_isError(actualSize)) {
std::cout << "Corrupted region chunk: " << path << ":" << pos.x << "." << pos.y << "." << pos.z << std::endl;
return false;
}
out.resize(actualSize);
return true;
}
void Region::save(const region_chunk_pos& pos, ZSTD_CCtx* cctx, const std::string_view& in) {
const auto maxSize = ZSTD_compressBound(in.size());
const auto buffer = new Region::data();
buffer->resize(maxSize);
const auto actualSize = ZSTD_compressCCtx(cctx, buffer->data(), buffer->capacity(), in.data(), in.size(), ZSTD_CLEVEL_DEFAULT);
if (ZSTD_isError(actualSize)) {
std::cout << "Corrupted chunk save: " << path << ":" << pos.x << "." << pos.y << "." << pos.z << std::endl;
return;
}
buffer->resize(actualSize);
{
std::unique_lock lock(mutex);
#ifndef LOW_MEMORY
{ // Save buffer
const auto it = content.find(pos);
if(it != content.end()) {
delete it->second;
content.erase(it);
}
content.insert({pos, buffer});
}
const auto& tmpPath = path;
#else
const auto tmpPath = path + ".tmp";
#endif
std::ofstream tmpFile(tmpPath, std::ios::out | std::ios::binary);
if (!tmpFile.good()) {
std::cout << "Corrupted region path: " << tmpPath << std::endl;
return;
}
// Write header
{
#ifdef LOW_MEMORY
ushort size = index.size() + 1;
#else
ushort size = (ushort)content.size();
#endif
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
}
#ifdef LOW_MEMORY
for(const auto& chunk: index) {
auto size = chunk.second.first;
#else
for(const auto& chunk: content) {
assert(chunk.second->size() < USHRT_MAX);
auto size = (ushort)chunk.second->size();
const auto out = chunk.second->data();
#endif
// Write pos
region_chunk_pos pos = chunk.first;
tmpFile.write(reinterpret_cast<char *>(&pos.x), sizeof(region_chunk_pos::value_type));
tmpFile.write(reinterpret_cast<char *>(&pos.y), sizeof(region_chunk_pos::value_type));
tmpFile.write(reinterpret_cast<char *>(&pos.z), sizeof(region_chunk_pos::value_type));
//NOTE: align uchar pos
if constexpr (sizeof(region_chunk_pos) % sizeof(ushort) != 0) {
tmpFile.put(0);
//MAYBE: store usefull uchar flags
}
// Write size
tmpFile.write(reinterpret_cast<char *>(&size), sizeof(size));
// Write content
#ifdef LOW_MEMORY
data tmp;
tmp.resize(size);
file.seekg(chunk.second.second);
const auto out = tmp.data();
file.read(out, size);
#endif
tmpFile.write(out, size);
}
if (!tmpFile.good()) {
std::cout << "region corrupted write " << tmpPath << std::endl;
tmpFile.close();
return;
}
tmpFile.close();
#ifdef LOW_MEMORY
index.clear();
file.close();
std::filesystem::rename(tmpPath, path);
load();
#endif
}
}

38
src/world/Region.hpp Normal file
View File

@ -0,0 +1,38 @@
#pragma once
#include "../data/glm.hpp"
#include <string>
#include <shared_mutex>
#include <zstd.h>
#include <robin_hood.h>
#include <fstream>
#define REGION_LENGTH 32
namespace world {
///Group of chunks saved as a single file
class Region {
public:
Region(const std::string& folderPath, const region_pos &pos);
~Region();
typedef std::vector<char> data;
bool read(const region_chunk_pos &pos, ZSTD_DCtx *dctx, data &out);
void save(const region_chunk_pos &pos, ZSTD_CCtx *cctx, const std::string_view &in);
private:
std::string path;
//TODO: try dictionnary
//TODO: use tickets to remove unused regions
std::shared_mutex mutex;
#ifdef LOW_MEMORY
std::ifstream file;
robin_hood::unordered_flat_map<region_chunk_pos, std::pair<ushort, std::streampos>> index;
#else
robin_hood::unordered_flat_map<region_chunk_pos, data*> content;
#endif
void load();
};
}

147
src/world/Universe.cpp
View File

@ -5,107 +5,112 @@
#include <Remotery.h>
#include <filesystem>
#include <fstream>
#include <zstd.h>
using namespace world;
Universe::Universe(const Universe::options &options): loadPool(4, options.folderPath), savePool(2, options.folderPath),
contouring(std::make_shared<contouring::Dummy>()) {
Universe::Universe(const Universe::options &options): generator(42), contouring(std::make_shared<contouring::Dummy>()) {
setOptions(options);
}
Universe::~Universe() {
contouring = NULL;
folderPath = options.folderPath;
struct vec_istream: std::streambuf {
vec_istream(std::vector<char> &vec) {
this->setg(&vec[0], &vec[0], &vec[0] + vec.size());
}
};
running = true;
std::filesystem::create_directories(folderPath);
// Save all
for(auto& pair: chunks) {
savePool.push(pair);
}
if(savePool.size() > 0) {
std::cout << "Saving..." << std::endl;
}
while(savePool.size() > 0) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
}
Universe::LoadPool::LoadPool(size_t count, const std::string& folderPath): folderPath(folderPath) {
for (size_t i = 0; i < count; i++) {
workers.push_back(std::thread([&] {
// Load workers
for (size_t i = 0; i < 4; i++) {
loadWorkers.push_back(std::thread([&] {
ZSTD_DCtx *dctx = ZSTD_createDCtx();
while (running) {
chunk_pos ctx;
loadQueue.wait();
if (loadQueue.pop(ctx)) {
//MAYBE: loadQueue.take to avoid duplicated work on fast move
rmt_ScopedCPUSample(ProcessLoad, 0);
const auto path = folderPath + '/' + std::to_string(ctx.x) + '.' +
std::to_string(ctx.y) + '.' + std::to_string(ctx.z) + ".map";
std::ifstream str(path);
if(str.good()) {
const region_pos rPos = glm::divide(ctx, region_chunk_pos(REGION_LENGTH));
const region_chunk_pos cPos = glm::modulo(ctx, region_chunk_pos(REGION_LENGTH));
const auto reg = getRegion(rPos);
Region::data data;
if(reg->read(cPos, dctx, data)) {
rmt_ScopedCPUSample(ProcessRead, 0);
loadedQueue.push({ctx, std::make_shared<Chunk>(str)});
if(str.bad()) {
std::cout << "Fail reading '" << path << "'" << std::endl;
}
vec_istream idata(data);
std::istream iss(&idata);
loadedQueue.push({ctx, std::make_shared<Chunk>(iss)});
} else {
rmt_ScopedCPUSample(ProcessGenerate, 0);
loadedQueue.push({ctx, std::make_shared<Chunk>(ctx, generator)});
}
}
}
ZSTD_freeDCtx(dctx);
}));
}
}
Universe::LoadPool::~LoadPool() {
running = false;
loadQueue.notify();
for (auto &worker : workers) {
if (worker.joinable())
worker.join();
}
}
inline void Universe::LoadPool::push(const chunk_pos &pos, int weight) { loadQueue.push(pos, weight); }
inline bool Universe::LoadPool::pop(robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> &out) { return loadedQueue.pop(out); }
inline size_t Universe::LoadPool::size() { return loadQueue.size(); }
Universe::SavePool::SavePool(size_t count, const std::string& folderPath): folderPath(folderPath) {
std::filesystem::create_directories(folderPath);
for (size_t i = 0; i < count; i++) {
workers.push_back(std::thread([&] {
// Save workers
for (size_t i = 0; i < 2; i++) {
saveWorkers.push_back(std::thread([&] {
ZSTD_CCtx* cctx = ZSTD_createCCtx();
while (running) {
robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> ctx;
queue.wait();
if (queue.pop(ctx) && ctx.second->isModified()) {
saveQueue.wait();
if (saveQueue.pop(ctx) && ctx.second->isModified()) {
//MAYBE: queue.take to avoid concurent write or duplicated work on fast move
rmt_ScopedCPUSample(ProcessSave, 0);
const auto path = folderPath + '/' + std::to_string(ctx.first.x) + '.' +
std::to_string(ctx.first.y) + '.' + std::to_string(ctx.first.z) + ".map";
std::ofstream ofs(path);
if(!ofs.good()) {
std::cout << "Fail opening '" << path << "' to save" << std::endl;
continue;
}
ctx.second->write(ofs);
ofs.flush();
ofs.close();
if(!ofs.good()) {
std::cout << "Fail closing '" << path << "' to save" << std::endl;
}
std::ostringstream out;
ctx.second->write(out);
const region_pos rPos = glm::divide(ctx.first, region_chunk_pos(REGION_LENGTH));
const region_chunk_pos cPos = glm::modulo(ctx.first, region_chunk_pos(REGION_LENGTH));
const auto reg = getRegion(rPos);
reg->save(cPos, cctx, out.str());
}
}
ZSTD_freeCCtx(cctx);
}));
}
}
Universe::SavePool::~SavePool() {
running = false;
queue.notify();
Universe::~Universe() {
contouring = NULL;
for (auto &worker : workers) {
// Save all
for(auto& pair: chunks) {
saveQueue.push(pair);
}
if(saveQueue.size() > 0) {
std::cout << "Saving..." << std::endl;
}
while(saveQueue.size() > 0) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
running = false;
loadQueue.notify();
saveQueue.notify();
for (auto &worker: loadWorkers) {
if (worker.joinable())
worker.join();
}
for (auto &worker: saveWorkers) {
if (worker.joinable())
worker.join();
}
}
inline void Universe::SavePool::push(const robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> &pos) { queue.push(pos); }
inline size_t Universe::SavePool::size() { return queue.size(); }
std::shared_ptr<Region> Universe::getRegion(const region_pos& pos) {
std::shared_lock lock(regionMutex);
const auto it = regionCache.find(pos);
if(it == regionCache.end()) {
lock.unlock();
const auto reg = std::make_shared<Region>(folderPath, pos);
std::unique_lock u_lock(regionMutex);
return regionCache.insert({pos, reg}).first->second;
} else {
return it->second;
}
}
void Universe::update(const camera_pos& pos, Universe::report& rep) {
const chunk_pos newPos = glm::divide(pos, chunk_voxel_pos(CHUNK_LENGTH));
@ -119,7 +124,7 @@ void Universe::update(const camera_pos& pos, Universe::report& rep) {
auto it = chunks.begin();
while (it != chunks.end()) {
if (glm::length2(last_pos - it->first) > keepDistance * keepDistance) {
savePool.push(*it);
saveQueue.push(*it);
it = chunks.erase(it);
} else {
if (const auto neighbors = it->second->update()) {
@ -131,7 +136,7 @@ void Universe::update(const camera_pos& pos, Universe::report& rep) {
}
}
}
rep.chunk_unload.push(savePool.size());
rep.chunk_unload.push(saveQueue.size());
{
rmt_ScopedCPUSample(Contouring, 0);
contouring->update(pos);
@ -149,18 +154,18 @@ void Universe::update(const camera_pos& pos, Universe::report& rep) {
if (dist2 <= loadDistance * loadDistance) {
const chunk_pos p = last_pos + glm::ivec3(x, y, z);
if (chunks.find(p) == chunks.end()) {
loadPool.push(p, -dist2);
loadQueue.push(p, -dist2);
}
}
}}}
}
rep.chunk_load.push(loadPool.size());
rep.chunk_load.push(loadQueue.size());
// Loaded chunks
{
rmt_ScopedCPUSample(Load, 0);
robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> loaded;
while (loadPool.pop(loaded)) {
while (loadedQueue.pop(loaded)) {
chunks.insert(loaded);
contouring->onUpdate(loaded.first, chunks, Faces::All);
}

50
src/world/Universe.hpp
View File

@ -7,7 +7,9 @@
#include "../data/safe_priority_queue.hpp"
#include "../data/circular_buffer.hpp"
#include "Chunk.hpp"
#include "Region.hpp"
#include "../data/geometry/Ray.hpp"
#include <shared_mutex>
#define REPORT_BUFFER_SIZE 128
@ -76,49 +78,25 @@ namespace world {
return {it->second};
}
/// Generating worker pool
class LoadPool {
public:
LoadPool(size_t size, const std::string& folderPath);
~LoadPool();
Generator generator;
Generator generator;
inline void push(const chunk_pos &pos, int weight);
inline bool pop(robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> &out);
inline size_t size();
bool running = true;
std::vector<std::thread> loadWorkers;
safe_priority_queue<chunk_pos, int> loadQueue;
safe_queue<robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>>> loadedQueue;
private:
std::string folderPath;
std::vector<std::thread> workers;
bool running = true;
safe_priority_queue<chunk_pos, int> loadQueue;
safe_queue<robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>>> loadedQueue;
};
LoadPool loadPool;
/// Write to file worker pool
class SavePool {
public:
SavePool(size_t size, const std::string& folderPath);
~SavePool();
inline void push(const robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>> &chunk);
inline size_t size();
private:
std::string folderPath;
std::vector<std::thread> workers;
bool running = true;
safe_queue<robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>>> queue; //NOTE: consider const Chunk
};
SavePool savePool;
std::vector<std::thread> saveWorkers;
safe_queue<robin_hood::pair<chunk_pos, std::shared_ptr<Chunk>>> saveQueue; //NOTE: consider const Chunk
int loadDistance;
int keepDistance;
std::string folderPath;
std::shared_mutex regionMutex; //MAYBE: shared_guard
robin_hood::unordered_map<region_pos, std::shared_ptr<Region>> regionCache;
//TODO: region pick list
std::shared_ptr<Region> getRegion(const region_pos &);
/// Contouring worker
std::shared_ptr<contouring::Abstract> contouring;
};

2
src/world/Voxel.hpp
View File

@ -6,6 +6,8 @@ namespace world {
/// Universe unit
struct Voxel {
/// Quantity of material
/// FIXME: low density area are cheatty
/// @note v < iso * UCHAR_MAX are useless
unsigned char Density;
/// Material type
/// @see world::materials