me
/
Univerxel
1
0
Fork 0
Code Releases Activity

Stash: Network compression

This commit is contained in:
May B. 2020-09-24 08:20:31 +02:00
parent ad73b02456
commit fd49ef3fa9
17 changed files with 250 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CMakeLists.txt
View File

@ -23,7 +23,9 @@ add_definitions(
)
add_subdirectory("include/glm")
add_subdirectory("include/enet")
set(LINKED_LIBS glfw pthread dl glm::glm_static enet)
add_subdirectory("include/zstd")
set(LINKED_LIBS glfw pthread dl
glm::glm_static enet::enet_static zstd::zstd_static)
file(GLOB_RECURSE SOURCES "src/*/*.cpp")
file(GLOB INCLUDE_SOURCES
@ -31,7 +33,6 @@ file(GLOB INCLUDE_SOURCES
"include/FastNoiseSIMD/*.cpp"
"include/tracy/TracyClient.cpp"
"include/meshoptimizer/*.cpp"
"include/zstd/*/*.c"
"include/gl3w/gl3w.c"
)
set(INCLUDE_LIBS
@ -42,9 +43,7 @@ set(INCLUDE_LIBS
"include/libguarded"
"include/tracy"
"include/meshoptimizer"
"include/zstd"
"include/gl3w"
"include/enet/include"
)
add_executable(univerxel "src/main.cpp" ${SOURCES} ${INCLUDE_SOURCES})

6
TODO.md
View File

@ -10,14 +10,16 @@
## Hello other
- [ ] Chat
- [~] Auth
- [~] Authentication
- [ ] Compression
- [ ] Encryption
- [x] Embedded
- [ ] Standalone
## Hello world
- [ ] Map stream
- [ ] Contouring
- [ ] Contouring service
- [ ] Edit
- [~] Occlusion Culling
- [ ] Iterator ray

16
include/enet/CMakeLists.txt
View File

@ -1,7 +1,3 @@
cmake_minimum_required(VERSION 2.6)
project(enet)
# The "configure" step.
include(CheckFunctionExists)
include(CheckStructHasMember)
@ -84,11 +80,13 @@ set(SOURCE_FILES
source_group(include FILES ${INCLUDE_FILES})
source_group(source FILES ${SOURCE_FILES})
add_library(enet STATIC
${INCLUDE_FILES}
${SOURCE_FILES}
)
add_library(enet INTERFACE)
target_include_directories(enet INTERFACE include)
add_library(enet_static STATIC ${INCLUDE_FILES} ${SOURCE_FILES})
target_link_libraries(enet_static PUBLIC enet)
add_library(enet::enet_static ALIAS enet_static)
if (MINGW)
target_link_libraries(enet winmm ws2_32)
target_link_libraries(enet_static winmm ws2_32)
endif()

12
include/zstd/CMakeLists.txt Normal file
View File

@ -0,0 +1,12 @@
file(GLOB INCLUDE_FILES "*/*.h")
file(GLOB SOURCE_FILES "*/*.c")
source_group(include FILES ${INCLUDE_FILES})
source_group(source FILES ${SOURCE_FILES})
add_library(zstd INTERFACE)
target_include_directories(zstd INTERFACE .)
add_library(zstd_static STATIC ${INCLUDE_FILES} ${SOURCE_FILES})
target_link_libraries(zstd_static PUBLIC zstd)
add_library(zstd::zstd_static ALIAS zstd_static)

26
src/client/world/DistanceUniverse.cpp
View File

@ -30,6 +30,32 @@ void DistantUniverse::update(voxel_pos pos, float) {
contouring->update(pos, areas);
}
void DistantUniverse::onData(enet_uint8 channel, ENetPacket* packet) {
const net::server_packet_type type = static_cast<net::server_packet_type>(*packet->data);
switch (type) {
case net::server_packet_type::COMPRESSION: {
if(dict.has_value())
break;
dict.emplace(packet->data + sizeof(net::server_packet_type), packet->dataLength - sizeof(net::server_packet_type));
LOG_D("Compression dictionnary loaded");
break;
}
case net::server_packet_type::CHUNK: {
if(!dict.has_value())
break;
LOG_D("Chunk !!!");
break;
}
default:
LOG_W("Bad packet from server");
break;
}
}
void DistantUniverse::emit(const action::packet &action) {
if(const auto move = std::get_if<action::Move>(&action)) {
send(net::client_packet_type::MOVE, move->pos, net::channel_type::NOTIFY);

6
src/client/world/DistantUniverse.hpp
View File

@ -2,6 +2,7 @@
#include "Universe.hpp"
#include "../../core/net/Client.hpp"
#include "../../core/utils/zctx.hpp"
namespace world::client {
class Area;
@ -17,10 +18,15 @@ namespace world::client {
ray_result raycast(const geometry::Ray &) const override;
void onData(enet_uint8 channel, ENetPacket* packet) override;
protected:
/// Alive areas containing chunks
area_map areas;
std::optional<zstd::read_dict_ctx> dict;
chunk_pos last_chunk = chunk_pos(INT_MAX);
};
}

31
src/core/data/file.hpp Normal file
View File

@ -0,0 +1,31 @@
#pragma once
#include <fstream>
#include <vector>
#include "../utils/logger.hpp"
namespace data {
class file_content: public std::vector<char> {
public:
// Read first
file_content(const std::vector<std::string>& paths): std::vector<char>() {
std::ifstream is = [&]() {
for(auto& path: paths) {
std::ifstream is(path, std::ios::in | std::ios::binary | std::ios::ate);
if(is.good()) {
return is;
}
is.close();
}
FATAL("File not found " << paths.back());
}();
const auto end = is.tellg();
is.seekg(0, std::ios::beg);
resize(end - is.tellg());
is.read(data(), size());
is.close();
}
};
}

69
src/core/data/zctx.hpp
View File

@ -1,69 +0,0 @@
#pragma once
#include <zstd.h>
#include <fstream>
#include <vector>
#include <cassert>
#include "../utils/logger.hpp"
namespace zstd {
struct read_ctx {
~read_ctx() {
ZSTD_freeDCtx(ctx);
}
ZSTD_DCtx *ctx;
ZSTD_DDict *dict;
};
struct write_ctx {
~write_ctx() {
ZSTD_freeCCtx(ctx);
}
ZSTD_CCtx *ctx;
ZSTD_CDict *dict;
};
class dict_set {
public:
dict_set(const void *data, size_t size) { load(data, size); }
dict_set(const std::vector<std::string>& paths) {
std::ifstream is = [&]() {
for(auto& path: paths) {
std::ifstream is(path, std::ios::in | std::ios::binary | std::ios::ate);
if(is.good()) {
return is;
}
is.close();
}
FATAL("Missing dict " << paths.back());
}();
const auto end = is.tellg();
is.seekg(0, std::ios::beg);
std::vector<char> dict(end - is.tellg());
is.read(dict.data(), dict.size());
is.close();
load(dict.data(), dict.size());
}
~dict_set() {
ZSTD_freeCDict(c);
ZSTD_freeDDict(d);
}
read_ctx make_reader() const {
return read_ctx{ZSTD_createDCtx(), d};
}
write_ctx make_writer() const {
return write_ctx{ZSTD_createCCtx(), c};
}
private:
void load(const void* data, size_t size) {
c = ZSTD_createCDict(data, size, ZSTD_CLEVEL_DEFAULT);
assert(c != NULL);
d = ZSTD_createDDict(data, size);
assert(d != NULL);
}
ZSTD_CDict *c;
ZSTD_DDict *d;
};
}

18
src/core/net/Server.hpp
View File

@ -61,6 +61,8 @@ public:
}
}
virtual const salt_t* getSalt(ENetPeer* peer) const { return peer->data; }
virtual void onConnect(ENetPeer* peer, enet_uint32 salt) {
LOG_D("Client connect from " << peer->address);
const salt_t rnd = std::rand();
@ -81,14 +83,18 @@ public:
}
bool sendTo(ENetPeer* peer, server_packet_type type, ENetPacket* packet, channel_type channel) {
assert(type < server_packet_type::CHUNK);
assert(packet->dataLength >= sizeof(server_packet_type) + sizeof(salt_t));
if(peer->data == NULL) {
LOG_W("Contacting " << peer->address << " before handshake");
return false;
if (type < server_packet_type::CHUNK) {
assert(packet->dataLength >= sizeof(server_packet_type) + sizeof(salt_t));
const auto salt = getSalt(peer);
if(salt == NULL) {
LOG_W("Contacting " << peer->address << " before handshake");
return false;
}
memcpy(packet->data + sizeof(server_packet_type), salt, sizeof(salt_t));
} else {
assert(packet->dataLength >= sizeof(server_packet_type));
}
memcpy(packet->data, &type, sizeof(server_packet_type));
memcpy(packet->data + sizeof(server_packet_type), peer->data, sizeof(salt_t));
return enet_peer_send(peer, (enet_uint8)channel, packet) == 0;
}
void broadcast(server_packet_type type, ENetPacket* packet, channel_type channel) {

4
src/core/net/data.hpp
View File

@ -24,6 +24,10 @@ enum class server_packet_type: enet_uint8 {
CHUNK = 16,
/// Chunk changed c0 realable
EDITS = 17,
/// World compression dictionary
/// data::file c0 realable?
COMPRESSION = 24,
};
enum class client_packet_type: enet_uint8 {
/// Interact with voxels

98
src/core/utils/zctx.hpp Normal file
View File

@ -0,0 +1,98 @@
#pragma once
#include <zstd.h>
#include <vector>
#include <cassert>
#include "logger.hpp"
namespace zstd {
/// Decompressor with ref to dictionnary
class read_ctx {
public:
read_ctx(ZSTD_DDict *dict): ctx(ZSTD_createDCtx()), dict(dict) { }
~read_ctx() {
ZSTD_freeDCtx(ctx);
}
/// Extract in to out (error message on failure)
template<typename I, typename O>
std::optional<const char*> decompress(const I& in, O& out) const {
const auto maxSize = ZSTD_getFrameContentSize(in.data(), in.size());
out.resize(maxSize);
const auto actualSize = ZSTD_decompress_usingDDict(ctx, out.data(), out.size(), in.data(), in.size(), dict);
if(ZSTD_isError(actualSize)) {
return ZSTD_getErrorName(actualSize);
}
out.resize(actualSize);
return std::nullopt;
}
protected:
ZSTD_DCtx *ctx;
ZSTD_DDict *dict;
};
/// Decompressor with builtin dictionnary
class read_dict_ctx: public read_ctx {
public:
read_dict_ctx(const void* data, size_t size): read_ctx(ZSTD_createDDict(data, size)) {
assert(dict != NULL);
}
~read_dict_ctx() {
ZSTD_freeDDict(dict);
}
};
/// Compressor with ref to dictionnary
class write_ctx {
public:
write_ctx(ZSTD_CDict *dict): ctx(ZSTD_createCCtx()), dict(dict) { }
~write_ctx() {
ZSTD_freeCCtx(ctx);
}
/// Compress in to out (error message on failure)
template<typename I, typename O>
std::optional<const char*> compress(const I& in, O& out) const {
const auto maxSize = ZSTD_compressBound(in.size());
out.resize(maxSize);
const auto actualSize = ZSTD_compress_usingCDict(ctx, out.data(), out.size(), in.data(), in.size(), dict);
if(ZSTD_isError(actualSize)) {
return ZSTD_getErrorName(actualSize);
}
out.resize(actualSize);
return std::nullopt;
}
protected:
ZSTD_CCtx *ctx;
ZSTD_CDict *dict;
};
class dict_set {
public:
dict_set(const std::vector<char>& data) { load(data.data(), data.size()); }
~dict_set() {
ZSTD_freeCDict(c);
ZSTD_freeDDict(d);
}
read_ctx make_reader() const {
return read_ctx(d);
}
write_ctx make_writer() const {
return write_ctx(c);
}
private:
void load(const void* data, size_t size) {
c = ZSTD_createCDict(data, size, ZSTD_CLEVEL_DEFAULT);
assert(c != NULL);
d = ZSTD_createDDict(data, size);
assert(d != NULL);
}
ZSTD_CDict *c;
ZSTD_DDict *d;
};
}

28
src/server/world/Universe.cpp
View File

@ -11,7 +11,9 @@ using namespace world::server;
const auto AREAS_FILE = "/areas.idx";
Universe::Universe(const Universe::options &options): net::Server(net::connection{"localhost", 4242}, 4), dicts({options.folderPath + "/zstd.dict", "content/zstd.dict"}) {
Universe::Universe(const Universe::options &options): net::Server(net::connection{"localhost", 4242}, 4),
dict_content({options.folderPath + "/zstd.dict", "content/zstd.dict"}), dicts(dict_content), dict_write_ctx(dicts.make_writer()) {
setOptions(options);
folderPath = options.folderPath;
struct vec_istream: std::streambuf {
@ -331,6 +333,30 @@ void Universe::onData(ENetPeer* peer, ENetPacket* packet, enet_uint8) {
break;
}
}
struct client_info {
};
void Universe::onConnect(ENetPeer* peer, enet_uint32 salt) {
LOG_D("Client connect from " << peer->address);
{
const salt_t rnd = std::rand();
peer->data = enet_malloc(sizeof(salt_t));
memcpy(peer->data, &salt, sizeof(salt_t));
const auto packet = enet_packet_create(NULL, sizeof(server_packet_type) + sizeof(salt_t) * 2, ENET_PACKET_FLAG_RELIABLE);
memcpy(packet->data + sizeof(server_packet_type) + sizeof(salt_t), &rnd, sizeof(salt_t));
sendTo(peer, server_packet_type::CHALLENGE, packet, net::channel_type::RELIABLE);
salt ^= rnd;
memcpy(peer->data, &salt, sizeof(salt_t));
}
{
const auto packet = enet_packet_create(NULL, sizeof(net::server_packet_type) + dict_content.size(), ENET_PACKET_FLAG_RELIABLE);
memcpy(packet->data + sizeof(net::server_packet_type), dict_content.data(), dict_content.size());
sendTo(peer, net::server_packet_type::COMPRESSION, packet, net::channel_type::RELIABLE);
}
}
const salt_t* Universe::getSalt(ENetPeer* peer) const {
return peer->data + offset(client_info::salt);
}
void Universe::updateChunk(area_map::iterator &, world::ChunkContainer::iterator &, chunk_pos, float deltaTime) {}
void Universe::loadChunk(area_<chunk_pos>, chunk_pos, const world::ChunkContainer &) {}

5
src/server/world/Universe.hpp
View File

@ -6,6 +6,7 @@
#include "../../core/data/math.hpp"
#include "../../core/data/safe_queue.hpp"
#include "../../core/data/safe_priority_queue.hpp"
#include "../../core/data/file.hpp"
#include "../../core/net/Server.hpp"
#include "Area.hpp"
@ -92,8 +93,12 @@ namespace world::server {
int keepDistance;
std::string folderPath;
data::file_content dict_content;
zstd::dict_set dicts;
zstd::write_ctx dict_write_ctx;
void onData(ENetPeer *peer, ENetPacket *, enet_uint8) override;
void onConnect(ENetPeer* peer, enet_uint32 salt) override;
const salt_t* getSalt(ENetPeer* peer) const override;
};
}

23
src/server/world/region/File.cpp
View File

@ -64,17 +64,14 @@ bool FileRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, da
if (it == index.end())
return false;
auto in = std::make_unique<data>();
in->resize(it->second.first);
data in;
in.resize(it->second.first);
file.seekg(it->second.second);
file.read(in->data(), in->size());
file.read(in.data(), in.size());
const auto maxSize = ZSTD_getFrameContentSize(in->data(), in->size());
out.resize(maxSize);
const auto actualSize = ZSTD_decompress_usingDDict(ctx.ctx, out.data(), out.size(), in->data(), in->size(), ctx.dict);
if(ZSTD_isError(actualSize)) {
if (auto err = ctx.decompress(in, out)) {
LOG_E("Corrupted region chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< ZSTD_getErrorName(actualSize));
<< err.value());
#ifdef REMOVE_CORRUPTED
LOG_W("Removing");
index.erase(it);
@ -83,21 +80,15 @@ bool FileRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx, da
#endif
return false;
}
out.resize(actualSize);
return true;
}
void FileRegion::write(const region_chunk_pos& pos, const zstd::write_ctx& ctx, const std::string_view& in) {
const auto maxSize = ZSTD_compressBound(in.size());
auto buffer = std::make_unique<FileRegion::data>();
buffer->resize(maxSize);
const auto actualSize = ZSTD_compress_usingCDict(ctx.ctx, buffer->data(), buffer->capacity(), in.data(), in.size(), ctx.dict);
if (ZSTD_isError(actualSize)) {
if (auto err = ctx.compress(in, *buffer)) {
LOG_E("Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< ZSTD_getErrorName(actualSize));
<< err.value());
return;
}
buffer->resize(actualSize);
save({{pos, std::move(buffer)}});
}

4
src/server/world/region/File.hpp
View File

@ -1,10 +1,10 @@
#pragma once
#include <vector>
#include <string>
#include <shared_mutex>
#include <fstream>
#include "../../../core/world/forward.h"
#include "../../../core/data/zctx.hpp"
#include "../../../core/utils/zctx.hpp"
#include "../../../core/data/math.hpp"
namespace world::server {

18
src/server/world/region/Memory.cpp
View File

@ -75,14 +75,9 @@ bool MemoryRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx,
if (it == content.end())
return false;
auto &in = it->second;
const auto maxSize = ZSTD_getFrameContentSize(in->data(), in->size());
out.resize(maxSize);
const auto actualSize = ZSTD_decompress_usingDDict(ctx.ctx, out.data(), out.size(), in->data(), in->size(), ctx.dict);
if(ZSTD_isError(actualSize)) {
if(auto err = ctx.decompress(*it->second, out)) {
LOG_E("Corrupted region chunk: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< ZSTD_getErrorName(actualSize));
<< err.value());
#ifdef REMOVE_CORRUPTED
LOG_W("Removing");
lock.unlock();
@ -94,21 +89,16 @@ bool MemoryRegion::read(const region_chunk_pos& pos, const zstd::read_ctx& ctx,
#endif
return false;
}
out.resize(actualSize);
return true;
}
void MemoryRegion::write(const region_chunk_pos& pos, const zstd::write_ctx& ctx, const std::string_view& in) {
const auto maxSize = ZSTD_compressBound(in.size());
const auto buffer = new MemoryRegion::data();
buffer->resize(maxSize);
const auto actualSize = ZSTD_compress_usingCDict(ctx.ctx, buffer->data(), buffer->capacity(), in.data(), in.size(), ctx.dict);
if (ZSTD_isError(actualSize)) {
if (auto err = ctx.compress(in, *buffer)) {
LOG_E("Corrupted chunk save: " << path << ":" << (int)pos.x << "." << (int)pos.y << "." << (int)pos.z << " "
<< ZSTD_getErrorName(actualSize));
<< err.value());
return;
}
buffer->resize(actualSize);
{
std::unique_lock lock(mutex);

5
src/server/world/region/Memory.hpp
View File

@ -1,11 +1,10 @@
#pragma once
#include <vector>
#include <string>
#include <shared_mutex>
#include <zstd.h>
#include <fstream>
#include "../../../core/world/forward.h"
#include "../../../core/data/zctx.hpp"
#include "../../../core/utils/zctx.hpp"
#include "../../../core/data/math.hpp"
namespace world::server {