Slow start chunk stream naive congestion
This commit is contained in:
parent
53968b611a
commit
fde2f4c64d
4
TODO.md
4
TODO.md
|
@ -36,6 +36,7 @@
|
||||||
- [~] Entities
|
- [~] Entities
|
||||||
- [ ] Collide
|
- [ ] Collide
|
||||||
- [ ] Get models
|
- [ ] Get models
|
||||||
|
- [ ] Reduce compile unit count
|
||||||
- [ ] Review documentation
|
- [ ] Review documentation
|
||||||
|
|
||||||
## Hello universe
|
## Hello universe
|
||||||
|
@ -77,7 +78,8 @@
|
||||||
- [ ] Break area part to entity
|
- [ ] Break area part to entity
|
||||||
- [ ] Slash screen
|
- [ ] Slash screen
|
||||||
- [ ] Start/Pause menu
|
- [ ] Start/Pause menu
|
||||||
- [ ] QUIC protocal
|
- [ ] QUIC protocol
|
||||||
|
- [ ] 8 for 1 contouring problem
|
||||||
- [ ] Use in memory protocol (to replace server_handle)
|
- [ ] Use in memory protocol (to replace server_handle)
|
||||||
- [ ] Octree
|
- [ ] Octree
|
||||||
- [ ] Better Lod
|
- [ ] Better Lod
|
||||||
|
|
|
@ -18,7 +18,7 @@ struct state {
|
||||||
std::optional<uint32_t> color;
|
std::optional<uint32_t> color;
|
||||||
};
|
};
|
||||||
struct {
|
struct {
|
||||||
std::array<char, 256> buffer;
|
std::array<char, 256> buffer = {'\0'};
|
||||||
std::vector<line> lines;
|
std::vector<line> lines;
|
||||||
} console;
|
} console;
|
||||||
};
|
};
|
||||||
|
|
|
@ -42,9 +42,10 @@ void DistantUniverse::update(voxel_pos pos, float deltaTime) {
|
||||||
++it_c;
|
++it_c;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (chunkChangeArea) { // Request missing chunks
|
if ((chunkChangeArea && (std::rand() & (1 << 4)-1) == 0) || mayQueryChunks) { // Request missing chunks
|
||||||
ZoneScopedN("Missing");
|
ZoneScopedN("Missing");
|
||||||
std::vector<chunk_pos> missing;
|
std::vector<chunk_pos> missing;
|
||||||
|
std::vector<long> missingDist;
|
||||||
//TODO: use easy sphere fill
|
//TODO: use easy sphere fill
|
||||||
const int queryDistance = std::min(loadDistance, serverDistance);
|
const int queryDistance = std::min(loadDistance, serverDistance);
|
||||||
for (int x = -queryDistance; x <= queryDistance; x++) {
|
for (int x = -queryDistance; x <= queryDistance; x++) {
|
||||||
|
@ -56,7 +57,23 @@ void DistantUniverse::update(voxel_pos pos, float deltaTime) {
|
||||||
if (chunks.find(p) != chunks.end()) {
|
if (chunks.find(p) != chunks.end()) {
|
||||||
contouring->onNotify(std::make_pair(area.first, p), diff, chunks);
|
contouring->onNotify(std::make_pair(area.first, p), diff, chunks);
|
||||||
} else {
|
} else {
|
||||||
missing.push_back(p);
|
if (missingDist.size() >= net::MAX_PENDING_CHUNK_COUNT) {
|
||||||
|
if (dist2 > missingDist.front())
|
||||||
|
continue;
|
||||||
|
missingDist.erase(missingDist.begin());
|
||||||
|
missing.erase(missing.begin());
|
||||||
|
}
|
||||||
|
|
||||||
|
auto it = missingDist.begin();
|
||||||
|
auto itv = missing.begin();
|
||||||
|
while (it != missingDist.end()) {
|
||||||
|
if (dist2 > *it)
|
||||||
|
break;
|
||||||
|
++it;
|
||||||
|
++itv;
|
||||||
|
}
|
||||||
|
missingDist.insert(it, dist2);
|
||||||
|
missing.insert(itv, p);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}}}
|
}}}
|
||||||
|
@ -67,6 +84,7 @@ void DistantUniverse::update(voxel_pos pos, float deltaTime) {
|
||||||
packet.write(missing.data(), missing.size() * sizeof(chunk_pos));
|
packet.write(missing.data(), missing.size() * sizeof(chunk_pos));
|
||||||
peer.send(packet.get(), net::channel_type::RELIABLE);
|
peer.send(packet.get(), net::channel_type::RELIABLE);
|
||||||
}
|
}
|
||||||
|
mayQueryChunks = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -93,6 +111,7 @@ void DistantUniverse::pullNetwork(voxel_pos pos) {
|
||||||
|
|
||||||
dict.emplace(packet->data + sizeof(server_packet_type), packet->dataLength - sizeof(server_packet_type));
|
dict.emplace(packet->data + sizeof(server_packet_type), packet->dataLength - sizeof(server_packet_type));
|
||||||
LOG_T("Compression dictionnary loaded");
|
LOG_T("Compression dictionnary loaded");
|
||||||
|
mayQueryChunks = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -134,6 +153,11 @@ void DistantUniverse::pullNetwork(voxel_pos pos) {
|
||||||
break;
|
break;
|
||||||
|
|
||||||
auto reader = PacketReader(packet, true);
|
auto reader = PacketReader(packet, true);
|
||||||
|
if (reader.isFull()) {
|
||||||
|
mayQueryChunks = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
area_<chunk_pos> pos;
|
area_<chunk_pos> pos;
|
||||||
if(!reader.read(pos))
|
if(!reader.read(pos))
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -48,6 +48,7 @@ namespace world::client {
|
||||||
std::optional<zstd::read_dict_ctx> dict;
|
std::optional<zstd::read_dict_ctx> dict;
|
||||||
|
|
||||||
chunk_pos last_chunk = chunk_pos(INT_MAX);
|
chunk_pos last_chunk = chunk_pos(INT_MAX);
|
||||||
|
bool mayQueryChunks = false;
|
||||||
|
|
||||||
ushort loadDistance;
|
ushort loadDistance;
|
||||||
ushort keepDistance;
|
ushort keepDistance;
|
||||||
|
|
|
@ -57,6 +57,7 @@ public:
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
#if TRACY_ENABLE
|
||||||
TracyPlot("SrvNetUpData", (int64_t)host->totalSentData);
|
TracyPlot("SrvNetUpData", (int64_t)host->totalSentData);
|
||||||
host->totalSentData = 0;
|
host->totalSentData = 0;
|
||||||
TracyPlot("SrvNetUpPackets", (int64_t)host->totalSentPackets);
|
TracyPlot("SrvNetUpPackets", (int64_t)host->totalSentPackets);
|
||||||
|
@ -65,6 +66,16 @@ public:
|
||||||
host->totalReceivedData = 0;
|
host->totalReceivedData = 0;
|
||||||
TracyPlot("SrvNetDownPackets", (int64_t)host->totalReceivedPackets);
|
TracyPlot("SrvNetDownPackets", (int64_t)host->totalReceivedPackets);
|
||||||
host->totalReceivedPackets = 0;
|
host->totalReceivedPackets = 0;
|
||||||
|
|
||||||
|
int64_t throttling = 0;
|
||||||
|
int64_t peerCount = 0;
|
||||||
|
iterPeers([&](peer_t *peer) {
|
||||||
|
throttling += ENET_PEER_PACKET_THROTTLE_SCALE - peer->packetThrottle;
|
||||||
|
peerCount++;
|
||||||
|
});
|
||||||
|
TracyPlot("SrvPeersCount", peerCount);
|
||||||
|
TracyPlot("SrvPeersThrottling", peerCount > 0 ? throttling * 100 / ENET_PEER_PACKET_THROTTLE_SCALE / peerCount : 0);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void disconnect(peer_t *peer, disconnect_reason reason) const {
|
void disconnect(peer_t *peer, disconnect_reason reason) const {
|
||||||
|
@ -74,6 +85,17 @@ public:
|
||||||
template<typename D>
|
template<typename D>
|
||||||
static D* GetPeerData(peer_t* peer) { return (D*)peer->data; }
|
static D* GetPeerData(peer_t* peer) { return (D*)peer->data; }
|
||||||
|
|
||||||
|
template<typename Call>
|
||||||
|
void iterPeers(Call call) {
|
||||||
|
for (auto currentPeer = host->peers;
|
||||||
|
currentPeer < &host->peers[host->peerCount];
|
||||||
|
++currentPeer)
|
||||||
|
{
|
||||||
|
if (currentPeer->state == ENET_PEER_STATE_CONNECTED)
|
||||||
|
call(currentPeer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Send to single peer
|
/// Send to single peer
|
||||||
/// Expect salt_t at peer->data
|
/// Expect salt_t at peer->data
|
||||||
bool sendTo(peer_t* peer, server_packet_type type, const void *data, size_t size, channel_type channel, std::optional<enet_uint32> flags = {}) {
|
bool sendTo(peer_t* peer, server_packet_type type, const void *data, size_t size, channel_type channel, std::optional<enet_uint32> flags = {}) {
|
||||||
|
|
|
@ -38,10 +38,11 @@ enum class server_packet_type: enet_uint8 {
|
||||||
AREAS = 16,
|
AREAS = 16,
|
||||||
/// Full chunk update
|
/// Full chunk update
|
||||||
/// {area_<chunk_pos>, zstd<chunk rle>} reliable
|
/// {area_<chunk_pos>, zstd<chunk rle>} reliable
|
||||||
|
/// empty: all sent reliable
|
||||||
CHUNK = 17,
|
CHUNK = 17,
|
||||||
/// Chunk changes
|
/// Chunk changes
|
||||||
/// {area_id, {chunk_pos, ushort(count), Chunk::Edit[]}[]} notify
|
/// {area_id, {chunk_pos, ushort(count), Chunk::Edit[]}[]} notify
|
||||||
/// FIXME: to big !!! MAYBE: compress
|
/// MAYBE: compress
|
||||||
EDITS = 18,
|
EDITS = 18,
|
||||||
|
|
||||||
/// Declare entities types
|
/// Declare entities types
|
||||||
|
@ -70,7 +71,7 @@ enum class client_packet_type: enet_uint8 {
|
||||||
FILL_SHAPE = 0,
|
FILL_SHAPE = 0,
|
||||||
|
|
||||||
/// Request missing chunks
|
/// Request missing chunks
|
||||||
/// area_id, chunk_pos[] reliable
|
/// area_id, chunk_pos[max: MAX_PENDING_CHUNK_COUNT] reliable
|
||||||
MISSING_CHUNKS = 8,
|
MISSING_CHUNKS = 8,
|
||||||
|
|
||||||
/// Send public text message
|
/// Send public text message
|
||||||
|
@ -82,6 +83,8 @@ enum class client_packet_type: enet_uint8 {
|
||||||
MOVE = 16,
|
MOVE = 16,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
constexpr auto MAX_PENDING_CHUNK_COUNT = 256;
|
||||||
|
|
||||||
struct connection {
|
struct connection {
|
||||||
std::string address;
|
std::string address;
|
||||||
int port;
|
int port;
|
||||||
|
|
|
@ -119,6 +119,46 @@ Universe::~Universe() {
|
||||||
LOG_D("Universe disappeared");
|
LOG_D("Universe disappeared");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct net_client {
|
||||||
|
net_client(net::salt_t salt, data::generational::id id): salt(salt), instanceId(id) { }
|
||||||
|
net::salt_t salt;
|
||||||
|
data::generational::id instanceId;
|
||||||
|
|
||||||
|
std::vector<std::pair<area_<chunk_pos>, long>> pendingChunks;
|
||||||
|
uint32_t chunkEmitRate = 0;
|
||||||
|
uint32_t chunkRateEpoch = 0;
|
||||||
|
bool popPendingChunk(area_<chunk_pos> &out) {
|
||||||
|
if (pendingChunks.empty())
|
||||||
|
return false;
|
||||||
|
|
||||||
|
out = pendingChunks.back().first;
|
||||||
|
pendingChunks.pop_back();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
void pushChunk(const area_<chunk_pos>& in, long dist) {
|
||||||
|
for (auto it = pendingChunks.begin(); it != pendingChunks.end(); ++it) {
|
||||||
|
if (it->first == in) {
|
||||||
|
pendingChunks.erase(it);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pendingChunks.size() >= net::MAX_PENDING_CHUNK_COUNT) {
|
||||||
|
if (dist > pendingChunks.front().second)
|
||||||
|
return;
|
||||||
|
pendingChunks.erase(pendingChunks.begin());
|
||||||
|
}
|
||||||
|
|
||||||
|
auto it = pendingChunks.begin();
|
||||||
|
while (it != pendingChunks.end()) {
|
||||||
|
if (dist > it->second)
|
||||||
|
break;
|
||||||
|
++it;
|
||||||
|
}
|
||||||
|
pendingChunks.insert(it, std::make_pair(in, dist));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
void Universe::saveAll(bool remove) {
|
void Universe::saveAll(bool remove) {
|
||||||
for(auto& area: areas) {
|
for(auto& area: areas) {
|
||||||
auto& chunks = area.second->setChunks();
|
auto& chunks = area.second->setChunks();
|
||||||
|
@ -371,19 +411,54 @@ void Universe::update(float deltaTime) {
|
||||||
for (auto handle = loadedQueue.extractor(); handle.first(loaded);) {
|
for (auto handle = loadedQueue.extractor(); handle.first(loaded);) {
|
||||||
if (const auto it = areas.find(loaded.first.first); it != areas.end()) {
|
if (const auto it = areas.find(loaded.first.first); it != areas.end()) {
|
||||||
it->second->setChunks().emplace(loaded.first.second, loaded.second);
|
it->second->setChunks().emplace(loaded.first.second, loaded.second);
|
||||||
loadChunk(loaded.first, glm::divide(it->second->getOffset().as_voxel()), it->second->getChunks());
|
const auto areaOffset = glm::divide(it->second->getOffset().as_voxel());
|
||||||
// MAYBE: limit chunks per update
|
loadChunk(loaded.first, areaOffset, it->second->getChunks());
|
||||||
host.broadcast(serializeChunk(loaded), net::channel_type::RELIABLE);
|
host.iterPeers([&](net::peer_t *peer) {
|
||||||
|
if (peer->data == nullptr)
|
||||||
|
return;
|
||||||
|
|
||||||
|
auto data = net::Server::GetPeerData<net_client>(peer);
|
||||||
|
if(auto entity = findEntity(PLAYER_ENTITY_ID, data->instanceId)) {
|
||||||
|
const auto dist = glm::length2(glm::divide(entity->pos.as_voxel()) - areaOffset - loaded.first.second);
|
||||||
|
if(dist <= glm::pow2(loadDistance))
|
||||||
|
data->pushChunk(loaded.first, dist);
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
host.iterPeers([&](net::peer_t *peer) {
|
||||||
|
if (peer->data == nullptr)
|
||||||
|
return;
|
||||||
|
|
||||||
struct net_client {
|
auto data = net::Server::GetPeerData<net_client>(peer);
|
||||||
net_client(net::salt_t salt, data::generational::id id): salt(salt), instanceId(id) { }
|
if (data->pendingChunks.empty())
|
||||||
net::salt_t salt;
|
return;
|
||||||
data::generational::id instanceId;
|
|
||||||
};
|
if (peer->packetThrottle > 3 * ENET_PEER_PACKET_THROTTLE_SCALE / 4)
|
||||||
|
data->chunkEmitRate = std::min(UINT32_MAX / 2, data->chunkEmitRate + 1 + (data->chunkEmitRate >> 6));
|
||||||
|
else if (peer->packetThrottleEpoch != data->chunkRateEpoch) {
|
||||||
|
data->chunkRateEpoch = peer->packetThrottleEpoch;
|
||||||
|
data->chunkEmitRate /= 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
area_<chunk_pos> pending;
|
||||||
|
size_t i = 0;
|
||||||
|
while(peer->outgoingDataTotal < data->chunkEmitRate && data->popPendingChunk(pending)) {
|
||||||
|
if (const auto it = areas.find(pending.first); it != areas.end()) {
|
||||||
|
if (const auto chunk = it->second->getChunks().findInRange(pending.second)) {
|
||||||
|
host.send(peer, serializeChunk(pending, std::dynamic_pointer_cast<Chunk>(chunk.value())), net::channel_type::RELIABLE);
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
peer->incomingBandwidth = 0;
|
||||||
|
peer->outgoingDataTotal = 0;
|
||||||
|
|
||||||
|
if (data->pendingChunks.empty())
|
||||||
|
host.send(peer, net::server_packet_type::CHUNK, nullptr, 0, net::channel_type::RELIABLE);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
void Universe::pullNetwork() {
|
void Universe::pullNetwork() {
|
||||||
ZoneScopedN("Network");
|
ZoneScopedN("Network");
|
||||||
|
@ -473,20 +548,21 @@ void Universe::pullNetwork() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case client_packet_type::MISSING_CHUNKS: {
|
case client_packet_type::MISSING_CHUNKS: {
|
||||||
if (auto player = findEntity(PLAYER_ENTITY_ID,Server::GetPeerData<net_client>(peer)->instanceId )) {
|
auto data = Server::GetPeerData<net_client>(peer);
|
||||||
|
if (auto player = findEntity(PLAYER_ENTITY_ID, data->instanceId )) {
|
||||||
const auto pos = player->pos.as_voxel();
|
const auto pos = player->pos.as_voxel();
|
||||||
|
|
||||||
auto reader = PacketReader(packet);
|
auto reader = PacketReader(packet);
|
||||||
area_id id = *reader.read<area_id>();
|
area_id id = *reader.read<area_id>();
|
||||||
if(auto area = areas.find(id); area != areas.end()) {
|
if(auto area = areas.find(id); area != areas.end()) {
|
||||||
auto &chunks = area->second->getChunks();
|
auto &chunks = area->second->getChunks();
|
||||||
const chunk_pos diff = glm::divide(pos - area->second->getOffset().as_voxel());
|
const chunk_pos areaOffset = glm::divide(pos - area->second->getOffset().as_voxel());
|
||||||
while(!reader.isFull()) {
|
|
||||||
chunk_pos cpos = *reader.read<chunk_pos>();
|
for (size_t i = 0; !reader.isFull() && i < MAX_PENDING_CHUNK_COUNT; i++) {
|
||||||
if(glm::length2(diff - cpos) <= glm::pow2(loadDistance) && chunks.inRange(cpos)) {
|
const chunk_pos cpos = *reader.read<chunk_pos>();
|
||||||
if(auto chunk = chunks.find(cpos); chunk != chunks.end()) {
|
const auto dist = glm::length2(areaOffset - cpos);
|
||||||
host.send(peer, serializeChunk({std::make_pair(id, cpos), std::dynamic_pointer_cast<Chunk>(chunk->second)}), net::channel_type::RELIABLE);
|
if (dist <= glm::pow2(loadDistance) && chunks.inRange(cpos) && chunks.findInRange(cpos).has_value()) {
|
||||||
}
|
data->pushChunk(std::make_pair(id, cpos), dist);
|
||||||
} else {
|
} else {
|
||||||
LOG_T("Request out of range chunk");
|
LOG_T("Request out of range chunk");
|
||||||
}
|
}
|
||||||
|
@ -516,13 +592,14 @@ void Universe::broadcastAreas() {
|
||||||
assert(packet.isFull());
|
assert(packet.isFull());
|
||||||
host.broadcast(packet.get(), net::channel_type::RELIABLE);
|
host.broadcast(packet.get(), net::channel_type::RELIABLE);
|
||||||
}
|
}
|
||||||
net::packet_t* Universe::serializeChunk(const robin_hood::pair<area_<chunk_pos>, std::shared_ptr<Chunk>> &pair) {
|
net::packet_t* Universe::serializeChunk(area_<chunk_pos> id, const std::shared_ptr<Chunk> &data) {
|
||||||
|
ZoneScopedN("Chunk");
|
||||||
std::ostringstream out;
|
std::ostringstream out;
|
||||||
pair.second->write(out);
|
data->write(out);
|
||||||
std::vector<char> buffer;
|
std::vector<char> buffer;
|
||||||
dict_write_ctx.compress(out.str(), buffer);
|
dict_write_ctx.compress(out.str(), buffer);
|
||||||
auto packet = net::Server::makePacket(net::server_packet_type::CHUNK, NULL, sizeof(pair.first) + buffer.size(), ENET_PACKET_FLAG_RELIABLE);
|
auto packet = net::Server::makePacket(net::server_packet_type::CHUNK, NULL, sizeof(id) + buffer.size(), ENET_PACKET_FLAG_RELIABLE);
|
||||||
packet.write(pair.first);
|
packet.write(id);
|
||||||
packet.write(buffer.data(), buffer.size());
|
packet.write(buffer.data(), buffer.size());
|
||||||
assert(packet.isFull());
|
assert(packet.isFull());
|
||||||
return packet.get();
|
return packet.get();
|
||||||
|
|
|
@ -69,7 +69,7 @@ namespace world::server {
|
||||||
/// Handle networking requests
|
/// Handle networking requests
|
||||||
void pullNetwork();
|
void pullNetwork();
|
||||||
void broadcastAreas();
|
void broadcastAreas();
|
||||||
net::packet_t* serializeChunk(const robin_hood::pair<area_<chunk_pos>, std::shared_ptr<Chunk>> &);
|
net::packet_t* serializeChunk(area_<chunk_pos>, const std::shared_ptr<Chunk>&);
|
||||||
virtual void broadcastMessage(const std::string &);
|
virtual void broadcastMessage(const std::string &);
|
||||||
|
|
||||||
using area_map = robin_hood::unordered_map<area_id, std::shared_ptr<Area>>;
|
using area_map = robin_hood::unordered_map<area_id, std::shared_ptr<Area>>;
|
||||||
|
|
Loading…
Reference in New Issue