From 67da78a1bc6ee3e6a3d4a21672df17a49ed572df Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Tue, 15 Oct 2024 16:39:18 -0700 Subject: [PATCH 01/26] [ntcore] LocalStorage: Move template functions inside class definition --- ntcore/src/main/native/cpp/LocalStorage.h | 66 +++++++++-------------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/ntcore/src/main/native/cpp/LocalStorage.h b/ntcore/src/main/native/cpp/LocalStorage.h index f4e15d404c1..007e2b51a60 100644 --- a/ntcore/src/main/native/cpp/LocalStorage.h +++ b/ntcore/src/main/native/cpp/LocalStorage.h @@ -246,13 +246,29 @@ class LocalStorage final : public net::ILocalStorage { template Timestamped::Value> GetAtomic( - NT_Handle subentry, typename TypeInfo::View defaultValue); + NT_Handle subentry, typename TypeInfo::View defaultValue) { + std::scoped_lock lock{m_mutex}; + Value* value = m_impl.GetSubEntryValue(subentry); + if (value && (IsNumericConvertibleTo(*value) || IsType(*value))) { + return GetTimestamped(*value); + } else { + return {0, 0, CopyValue(defaultValue)}; + } + } template Timestamped::SmallRet> GetAtomic( NT_Handle subentry, wpi::SmallVectorImpl::SmallElem>& buf, - typename TypeInfo::View defaultValue); + typename TypeInfo::View defaultValue) { + std::scoped_lock lock{m_mutex}; + Value* value = m_impl.GetSubEntryValue(subentry); + if (value && (IsNumericConvertibleTo(*value) || IsType(*value))) { + return GetTimestamped(*value, buf); + } else { + return {0, 0, CopyValue(defaultValue, buf)}; + } + } std::vector ReadQueueValue(NT_Handle subentry, unsigned int types) { std::scoped_lock lock{m_mutex}; @@ -265,7 +281,14 @@ class LocalStorage final : public net::ILocalStorage { template std::vector::Value>> ReadQueue( - NT_Handle subentry); + NT_Handle subentry) { + std::scoped_lock lock{m_mutex}; + auto subscriber = m_impl.GetSubEntry(subentry); + if (!subscriber) { + return {}; + } + return subscriber->pollStorage.Read(); + } // // Backwards compatible user functions @@ -665,41 +688,4 @@ class LocalStorage final : public net::ILocalStorage { Impl m_impl; }; -template -Timestamped::Value> LocalStorage::GetAtomic( - NT_Handle subentry, typename TypeInfo::View defaultValue) { - std::scoped_lock lock{m_mutex}; - Value* value = m_impl.GetSubEntryValue(subentry); - if (value && (IsNumericConvertibleTo(*value) || IsType(*value))) { - return GetTimestamped(*value); - } else { - return {0, 0, CopyValue(defaultValue)}; - } -} - -template -Timestamped::SmallRet> LocalStorage::GetAtomic( - NT_Handle subentry, - wpi::SmallVectorImpl::SmallElem>& buf, - typename TypeInfo::View defaultValue) { - std::scoped_lock lock{m_mutex}; - Value* value = m_impl.GetSubEntryValue(subentry); - if (value && (IsNumericConvertibleTo(*value) || IsType(*value))) { - return GetTimestamped(*value, buf); - } else { - return {0, 0, CopyValue(defaultValue, buf)}; - } -} - -template -std::vector::Value>> LocalStorage::ReadQueue( - NT_Handle subentry) { - std::scoped_lock lock{m_mutex}; - auto subscriber = m_impl.GetSubEntry(subentry); - if (!subscriber) { - return {}; - } - return subscriber->pollStorage.Read(); -} - } // namespace nt From 4d48b28368285834cb656567d34e0e91d136f23a Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Tue, 15 Oct 2024 16:44:34 -0700 Subject: [PATCH 02/26] [ntcore] Value: Inline constructors --- ntcore/src/main/native/cpp/Value.cpp | 30 -------------- .../include/networktables/NetworkTableValue.h | 40 +++++++++++++++++-- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/ntcore/src/main/native/cpp/Value.cpp b/ntcore/src/main/native/cpp/Value.cpp index c54a2ac63ea..6d1e73bcd11 100644 --- a/ntcore/src/main/native/cpp/Value.cpp +++ b/ntcore/src/main/native/cpp/Value.cpp @@ -18,7 +18,6 @@ #include "Value_internal.h" #include "networktables/NetworkTableValue.h" -#include "ntcore_cpp.h" using namespace nt; @@ -70,35 +69,6 @@ void StringArrayStorage::InitNtStrings() { } } -Value::Value() { - m_val.type = NT_UNASSIGNED; - m_val.last_change = 0; - m_val.server_time = 0; - m_size = 0; -} - -Value::Value(NT_Type type, size_t size, int64_t time, const private_init&) - : Value{type, size, time == 0 ? nt::Now() : time, 1, private_init{}} {} - -Value::Value(NT_Type type, size_t size, int64_t time, int64_t serverTime, - const private_init&) { - m_val.type = type; - m_val.last_change = time; - m_val.server_time = serverTime; - if (m_val.type == NT_BOOLEAN_ARRAY) { - m_val.data.arr_boolean.arr = nullptr; - } else if (m_val.type == NT_INTEGER_ARRAY) { - m_val.data.arr_int.arr = nullptr; - } else if (m_val.type == NT_FLOAT_ARRAY) { - m_val.data.arr_float.arr = nullptr; - } else if (m_val.type == NT_DOUBLE_ARRAY) { - m_val.data.arr_double.arr = nullptr; - } else if (m_val.type == NT_STRING_ARRAY) { - m_val.data.arr_string.arr = nullptr; - } - m_size = size; -} - Value Value::MakeBooleanArray(std::span value, int64_t time) { Value val{NT_BOOLEAN_ARRAY, value.size() * sizeof(int), time, private_init{}}; auto data = AllocateArray(value.size()); diff --git a/ntcore/src/main/native/include/networktables/NetworkTableValue.h b/ntcore/src/main/native/include/networktables/NetworkTableValue.h index 80cee51f41e..2dcd65e9794 100644 --- a/ntcore/src/main/native/include/networktables/NetworkTableValue.h +++ b/ntcore/src/main/native/include/networktables/NetworkTableValue.h @@ -20,6 +20,9 @@ namespace nt { +// Forward declare here to avoid circular dependency on ntcore_cpp.h +int64_t Now(); + #if __GNUC__ >= 13 #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" @@ -33,10 +36,41 @@ class Value final { struct private_init {}; public: - Value(); - Value(NT_Type type, size_t size, int64_t time, const private_init&); + Value() { + m_val.type = NT_UNASSIGNED; + m_val.last_change = 0; + m_val.server_time = 0; + } + + Value(NT_Type type, size_t size, int64_t time, const private_init&) + : Value{type, size, time == 0 ? Now() : time, 1, private_init{}} {} + Value(NT_Type type, size_t size, int64_t time, int64_t serverTime, - const private_init&); + const private_init&) + : m_size{size} { + m_val.type = type; + m_val.last_change = time; + m_val.server_time = serverTime; + switch (type) { + case NT_BOOLEAN_ARRAY: + m_val.data.arr_boolean.arr = nullptr; + break; + case NT_INTEGER_ARRAY: + m_val.data.arr_int.arr = nullptr; + break; + case NT_FLOAT_ARRAY: + m_val.data.arr_float.arr = nullptr; + break; + case NT_DOUBLE_ARRAY: + m_val.data.arr_double.arr = nullptr; + break; + case NT_STRING_ARRAY: + m_val.data.arr_string.arr = nullptr; + break; + default: + break; + } + } explicit operator bool() const { return m_val.type != NT_UNASSIGNED; } From 4d362be7651354561b4aa055fbece9323a4ae688 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Tue, 15 Oct 2024 16:48:10 -0700 Subject: [PATCH 03/26] [ntcore] NetworkOutgoingQueue: Move function defs inside class --- .../native/cpp/net/NetworkOutgoingQueue.h | 393 +++++++++--------- 1 file changed, 189 insertions(+), 204 deletions(-) diff --git a/ntcore/src/main/native/cpp/net/NetworkOutgoingQueue.h b/ntcore/src/main/native/cpp/net/NetworkOutgoingQueue.h index 16950ce8bbd..5f4330216a2 100644 --- a/ntcore/src/main/native/cpp/net/NetworkOutgoingQueue.h +++ b/ntcore/src/main/native/cpp/net/NetworkOutgoingQueue.h @@ -69,7 +69,43 @@ class NetworkOutgoingQueue { m_queues.emplace_back(100); // default queue is 100 ms period } - void SetPeriod(int id, uint32_t periodMs); + void SetPeriod(int id, uint32_t periodMs) { + // it's quite common to set a lot of things in a row with the same period + unsigned int queueIndex; + if (m_lastSetPeriod == periodMs) { + queueIndex = m_lastSetPeriodQueueIndex; + } else { + // find and possibly create queue for this period + auto it = + std::find_if(m_queues.begin(), m_queues.end(), + [&](const auto& q) { return q.periodMs == periodMs; }); + if (it == m_queues.end()) { + queueIndex = m_queues.size(); + m_queues.emplace_back(periodMs); + } else { + queueIndex = it - m_queues.begin(); + } + m_lastSetPeriodQueueIndex = queueIndex; + m_lastSetPeriod = periodMs; + } + + // map the handle to the queue + auto [infoIt, created] = m_idMap.try_emplace(id); + if (!created && infoIt->getSecond().queueIndex != queueIndex) { + // need to move any items from old queue to new queue + auto& oldMsgs = m_queues[infoIt->getSecond().queueIndex].msgs; + auto it = + std::stable_partition(oldMsgs.begin(), oldMsgs.end(), + [&](const auto& e) { return e.id != id; }); + auto& newMsgs = m_queues[queueIndex].msgs; + for (auto i = it, end = oldMsgs.end(); i != end; ++i) { + newMsgs.emplace_back(std::move(*i)); + } + oldMsgs.erase(it, oldMsgs.end()); + } + + infoIt->getSecond().queueIndex = queueIndex; + } void EraseId(int id) { m_idMap.erase(id); } @@ -79,9 +115,146 @@ class NetworkOutgoingQueue { m_totalSize += sizeof(Message); } - void SendValue(int id, const Value& value, ValueSendMode mode); + void SendValue(int id, const Value& value, ValueSendMode mode) { + if (m_local) { + mode = ValueSendMode::kImm; // always send local immediately + } + // backpressure by stopping sending all if the buffer is too full + if (mode == ValueSendMode::kAll && m_totalSize >= kOutgoingLimit) { + mode = ValueSendMode::kNormal; + } + switch (mode) { + case ValueSendMode::kDisabled: // do nothing + break; + case ValueSendMode::kImm: // send immediately + m_wire.SendBinary([&](auto& os) { EncodeValue(os, id, value); }); + break; + case ValueSendMode::kAll: { // append to outgoing + auto& info = m_idMap[id]; + auto& queue = m_queues[info.queueIndex]; + info.valuePos = queue.msgs.size(); + queue.Append(id, ValueMsg{id, value}); + m_totalSize += sizeof(Message) + value.size(); + break; + } + case ValueSendMode::kNormal: { + // replace, or append if not present + auto& info = m_idMap[id]; + auto& queue = m_queues[info.queueIndex]; + if (info.valuePos != -1 && + static_cast(info.valuePos) < queue.msgs.size()) { + auto& elem = queue.msgs[info.valuePos]; + if (auto m = std::get_if(&elem.msg.contents)) { + // double-check handle, and only replace if timestamp newer + if (elem.id == id && + (m->value.time() == 0 || value.time() >= m->value.time())) { + int delta = value.size() - m->value.size(); + m->value = value; + m_totalSize += delta; + return; + } + } + } + info.valuePos = queue.msgs.size(); + queue.Append(id, ValueMsg{id, value}); + m_totalSize += sizeof(Message) + value.size(); + break; + } + } + } + + void SendOutgoing(uint64_t curTimeMs, bool flush) { + if (m_totalSize == 0) { + return; // nothing to do + } + + // rate limit frequency of transmissions + if (curTimeMs < (m_lastSendMs + kMinPeriodMs)) { + return; + } + + if (!m_wire.Ready()) { + return; // don't bother, still sending the last batch + } - void SendOutgoing(uint64_t curTimeMs, bool flush); + // what queues are ready to send? + wpi::SmallVector queues; + for (unsigned int i = 0; i < m_queues.size(); ++i) { + if (!m_queues[i].msgs.empty() && + (flush || curTimeMs >= m_queues[i].nextSendMs)) { + queues.emplace_back(i); + } + } + if (queues.empty()) { + return; // nothing needs to be sent yet + } + + // Sort transmission order by what queue has been waiting the longest time. + // XXX: byte-weighted fair queueing might be better, but is much more + // complex to implement. + std::sort(queues.begin(), queues.end(), [&](const auto& a, const auto& b) { + return m_queues[a].nextSendMs < m_queues[b].nextSendMs; + }); + + for (unsigned int queueIndex : queues) { + auto& queue = m_queues[queueIndex]; + auto& msgs = queue.msgs; + auto it = msgs.begin(); + auto end = msgs.end(); + int unsent = 0; + for (; it != end && unsent == 0; ++it) { + if (auto m = std::get_if(&it->msg.contents)) { + unsent = m_wire.WriteBinary( + [&](auto& os) { EncodeValue(os, it->id, m->value); }); + } else { + unsent = m_wire.WriteText([&](auto& os) { + if (!WireEncodeText(os, it->msg)) { + os << "{}"; + } + }); + } + } + if (unsent < 0) { + return; // error + } + if (unsent == 0) { + // finish writing any partial buffers + unsent = m_wire.Flush(); + if (unsent < 0) { + return; // error + } + } + int delta = it - msgs.begin() - unsent; + for (auto&& msg : std::span{msgs}.subspan(0, delta)) { + if (auto m = std::get_if(&msg.msg.contents)) { + m_totalSize -= sizeof(Message) + m->value.size(); + } else { + m_totalSize -= sizeof(Message); + } + } + msgs.erase(msgs.begin(), it - unsent); + for (auto&& kv : m_idMap) { + auto& info = kv.getSecond(); + if (info.queueIndex == queueIndex) { + if (info.valuePos < delta) { + info.valuePos = -1; + } else { + info.valuePos -= delta; + } + } + } + + // try to stay on periodic timing, unless it's falling behind current time + if (unsent == 0) { + queue.nextSendMs += queue.periodMs; + if (queue.nextSendMs < curTimeMs) { + queue.nextSendMs = curTimeMs + queue.periodMs; + } + } + } + + m_lastSendMs = curTimeMs; + } void SetTimeOffset(int64_t offsetUs) { m_timeOffsetUs = offsetUs; } int64_t GetTimeOffset() const { return m_timeOffsetUs; } @@ -92,7 +265,19 @@ class NetworkOutgoingQueue { private: using ValueMsg = typename MessageType::ValueMsg; - void EncodeValue(wpi::raw_ostream& os, int id, const Value& value); + void EncodeValue(wpi::raw_ostream& os, int id, const Value& value) { + int64_t time = value.time(); + if constexpr (std::same_as) { + if (time != 0) { + time += m_timeOffsetUs; + // make sure resultant time isn't exactly 0 + if (time == 0) { + time = 1; + } + } + } + WireEncodeBinary(os, id, time, value); + } struct Message { Message() = default; @@ -132,204 +317,4 @@ class NetworkOutgoingQueue { static constexpr size_t kOutgoingLimit = 1024 * 1024; }; -template -void NetworkOutgoingQueue::SetPeriod(int id, uint32_t periodMs) { - // it's quite common to set a lot of things in a row with the same period - unsigned int queueIndex; - if (m_lastSetPeriod == periodMs) { - queueIndex = m_lastSetPeriodQueueIndex; - } else { - // find and possibly create queue for this period - auto it = - std::find_if(m_queues.begin(), m_queues.end(), - [&](const auto& q) { return q.periodMs == periodMs; }); - if (it == m_queues.end()) { - queueIndex = m_queues.size(); - m_queues.emplace_back(periodMs); - } else { - queueIndex = it - m_queues.begin(); - } - m_lastSetPeriodQueueIndex = queueIndex; - m_lastSetPeriod = periodMs; - } - - // map the handle to the queue - auto [infoIt, created] = m_idMap.try_emplace(id); - if (!created && infoIt->getSecond().queueIndex != queueIndex) { - // need to move any items from old queue to new queue - auto& oldMsgs = m_queues[infoIt->getSecond().queueIndex].msgs; - auto it = std::stable_partition(oldMsgs.begin(), oldMsgs.end(), - [&](const auto& e) { return e.id != id; }); - auto& newMsgs = m_queues[queueIndex].msgs; - for (auto i = it, end = oldMsgs.end(); i != end; ++i) { - newMsgs.emplace_back(std::move(*i)); - } - oldMsgs.erase(it, oldMsgs.end()); - } - - infoIt->getSecond().queueIndex = queueIndex; -} - -template -void NetworkOutgoingQueue::SendValue(int id, const Value& value, - ValueSendMode mode) { - if (m_local) { - mode = ValueSendMode::kImm; // always send local immediately - } - // backpressure by stopping sending all if the buffer is too full - if (mode == ValueSendMode::kAll && m_totalSize >= kOutgoingLimit) { - mode = ValueSendMode::kNormal; - } - switch (mode) { - case ValueSendMode::kDisabled: // do nothing - break; - case ValueSendMode::kImm: // send immediately - m_wire.SendBinary([&](auto& os) { EncodeValue(os, id, value); }); - break; - case ValueSendMode::kAll: { // append to outgoing - auto& info = m_idMap[id]; - auto& queue = m_queues[info.queueIndex]; - info.valuePos = queue.msgs.size(); - queue.Append(id, ValueMsg{id, value}); - m_totalSize += sizeof(Message) + value.size(); - break; - } - case ValueSendMode::kNormal: { - // replace, or append if not present - auto& info = m_idMap[id]; - auto& queue = m_queues[info.queueIndex]; - if (info.valuePos != -1 && - static_cast(info.valuePos) < queue.msgs.size()) { - auto& elem = queue.msgs[info.valuePos]; - if (auto m = std::get_if(&elem.msg.contents)) { - // double-check handle, and only replace if timestamp newer - if (elem.id == id && - (m->value.time() == 0 || value.time() >= m->value.time())) { - int delta = value.size() - m->value.size(); - m->value = value; - m_totalSize += delta; - return; - } - } - } - info.valuePos = queue.msgs.size(); - queue.Append(id, ValueMsg{id, value}); - m_totalSize += sizeof(Message) + value.size(); - break; - } - } -} - -template -void NetworkOutgoingQueue::SendOutgoing(uint64_t curTimeMs, - bool flush) { - if (m_totalSize == 0) { - return; // nothing to do - } - - // rate limit frequency of transmissions - if (curTimeMs < (m_lastSendMs + kMinPeriodMs)) { - return; - } - - if (!m_wire.Ready()) { - return; // don't bother, still sending the last batch - } - - // what queues are ready to send? - wpi::SmallVector queues; - for (unsigned int i = 0; i < m_queues.size(); ++i) { - if (!m_queues[i].msgs.empty() && - (flush || curTimeMs >= m_queues[i].nextSendMs)) { - queues.emplace_back(i); - } - } - if (queues.empty()) { - return; // nothing needs to be sent yet - } - - // Sort transmission order by what queue has been waiting the longest time. - // XXX: byte-weighted fair queueing might be better, but is much more complex - // to implement. - std::sort(queues.begin(), queues.end(), [&](const auto& a, const auto& b) { - return m_queues[a].nextSendMs < m_queues[b].nextSendMs; - }); - - for (unsigned int queueIndex : queues) { - auto& queue = m_queues[queueIndex]; - auto& msgs = queue.msgs; - auto it = msgs.begin(); - auto end = msgs.end(); - int unsent = 0; - for (; it != end && unsent == 0; ++it) { - if (auto m = std::get_if(&it->msg.contents)) { - unsent = m_wire.WriteBinary( - [&](auto& os) { EncodeValue(os, it->id, m->value); }); - } else { - unsent = m_wire.WriteText([&](auto& os) { - if (!WireEncodeText(os, it->msg)) { - os << "{}"; - } - }); - } - } - if (unsent < 0) { - return; // error - } - if (unsent == 0) { - // finish writing any partial buffers - unsent = m_wire.Flush(); - if (unsent < 0) { - return; // error - } - } - int delta = it - msgs.begin() - unsent; - for (auto&& msg : std::span{msgs}.subspan(0, delta)) { - if (auto m = std::get_if(&msg.msg.contents)) { - m_totalSize -= sizeof(Message) + m->value.size(); - } else { - m_totalSize -= sizeof(Message); - } - } - msgs.erase(msgs.begin(), it - unsent); - for (auto&& kv : m_idMap) { - auto& info = kv.getSecond(); - if (info.queueIndex == queueIndex) { - if (info.valuePos < delta) { - info.valuePos = -1; - } else { - info.valuePos -= delta; - } - } - } - - // try to stay on periodic timing, unless it's falling behind current time - if (unsent == 0) { - queue.nextSendMs += queue.periodMs; - if (queue.nextSendMs < curTimeMs) { - queue.nextSendMs = curTimeMs + queue.periodMs; - } - } - } - - m_lastSendMs = curTimeMs; -} - -template -void NetworkOutgoingQueue::EncodeValue(wpi::raw_ostream& os, - int id, - const Value& value) { - int64_t time = value.time(); - if constexpr (std::same_as) { - if (time != 0) { - time += m_timeOffsetUs; - // make sure resultant time isn't exactly 0 - if (time == 0) { - time = 1; - } - } - } - WireEncodeBinary(os, id, time, value); -} - } // namespace nt::net From 5ec504ef50aa0f9cedf75e0eaf2146db89caf156 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Tue, 15 Oct 2024 17:03:12 -0700 Subject: [PATCH 04/26] [ntcore] Move ServerImpl to nt::server namespace --- ntcore/CMakeLists.txt | 1 + ntcore/src/main/native/cpp/NetworkServer.h | 4 +- .../native/cpp/{net => server}/ServerImpl.cpp | 85 +++++++++---------- .../native/cpp/{net => server}/ServerImpl.h | 77 +++++++++-------- .../cpp/{net => server}/ServerImplTest.cpp | 10 +-- 5 files changed, 91 insertions(+), 86 deletions(-) rename ntcore/src/main/native/cpp/{net => server}/ServerImpl.cpp (95%) rename ntcore/src/main/native/cpp/{net => server}/ServerImpl.h (90%) rename ntcore/src/test/native/cpp/{net => server}/ServerImplTest.cpp (98%) diff --git a/ntcore/CMakeLists.txt b/ntcore/CMakeLists.txt index 94bfbe95b8f..c8fdca2262b 100644 --- a/ntcore/CMakeLists.txt +++ b/ntcore/CMakeLists.txt @@ -10,6 +10,7 @@ file( src/main/native/cpp/net/*.cpp src/main/native/cpp/net3/*.cpp src/main/native/cpp/networktables/*.cpp + src/main/native/cpp/server/*.cpp src/main/native/cpp/tables/*.cpp ) add_library(ntcore ${ntcore_native_src}) diff --git a/ntcore/src/main/native/cpp/NetworkServer.h b/ntcore/src/main/native/cpp/NetworkServer.h index db4ecfac057..14e7dc95bae 100644 --- a/ntcore/src/main/native/cpp/NetworkServer.h +++ b/ntcore/src/main/native/cpp/NetworkServer.h @@ -18,8 +18,8 @@ #include "net/ClientMessageQueue.h" #include "net/Message.h" -#include "net/ServerImpl.h" #include "ntcore_cpp.h" +#include "server/ServerImpl.h" namespace wpi { class Logger; @@ -78,7 +78,7 @@ class NetworkServer { using Queue = net::LocalClientMessageQueue; net::ClientMessage m_localMsgs[Queue::kBlockSize]; - net::ServerImpl m_serverImpl; + server::ServerImpl m_serverImpl; // shared with user (must be atomic or mutex-protected) std::atomic*> m_flushLocalAtomic{nullptr}; diff --git a/ntcore/src/main/native/cpp/net/ServerImpl.cpp b/ntcore/src/main/native/cpp/server/ServerImpl.cpp similarity index 95% rename from ntcore/src/main/native/cpp/net/ServerImpl.cpp rename to ntcore/src/main/native/cpp/server/ServerImpl.cpp index 016929bbae4..4d7c9fb5bca 100644 --- a/ntcore/src/main/native/cpp/net/ServerImpl.cpp +++ b/ntcore/src/main/native/cpp/server/ServerImpl.cpp @@ -23,9 +23,7 @@ #include #include -#include "IConnectionList.h" #include "Log.h" -#include "NetworkInterface.h" #include "Types_internal.h" #include "net/Message.h" #include "net/WireEncoder.h" @@ -35,7 +33,7 @@ #include "ntcore_c.h" using namespace nt; -using namespace nt::net; +using namespace nt::server; using namespace mpack; // maximum amount of time the wire can be not ready to send another @@ -293,7 +291,7 @@ void ServerImpl::ClientData4Base::ClientSubscribe( // update periodic sender (if not local) if (!m_local) { - m_periodMs = UpdatePeriodCalc(m_periodMs, sub->periodMs); + m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->periodMs); m_setPeriodic(m_periodMs); } @@ -312,7 +310,7 @@ void ServerImpl::ClientData4Base::ClientSubscribe( bool wasSubscribed = tcdIt != topic->clients.end() && !tcdIt->second.subscribers.empty(); bool wasSubscribedValue = - wasSubscribed ? tcdIt->second.sendMode != ValueSendMode::kDisabled + wasSubscribed ? tcdIt->second.sendMode != net::ValueSendMode::kDisabled : false; bool added = false; @@ -344,7 +342,7 @@ void ServerImpl::ClientData4Base::ClientSubscribe( for (auto topic : dataToSend) { DEBUG4("send last value for {} to client {}", topic->name, m_id); - SendValue(topic, topic->lastValue, ValueSendMode::kAll); + SendValue(topic, topic->lastValue, net::ValueSendMode::kAll); } } @@ -372,7 +370,7 @@ void ServerImpl::ClientData4Base::ClientUnsubscribe(int subuid) { // loop over all subscribers to update period if (!m_local) { - m_periodMs = CalculatePeriod( + m_periodMs = net::CalculatePeriod( m_subscribers, [](auto& x) { return x.getSecond()->periodMs; }); m_setPeriodic(m_periodMs); } @@ -392,7 +390,7 @@ void ServerImpl::ClientData4Base::ClientSetValue(int pubuid, void ServerImpl::ClientDataLocal::SendValue(TopicData* topic, const Value& value, - ValueSendMode mode) { + net::ValueSendMode mode) { if (m_server.m_local) { m_server.m_local->ServerSetValue(topic->localTopic, value); } @@ -435,10 +433,10 @@ void ServerImpl::ClientDataLocal::SendPropertiesUpdate(TopicData* topic, } bool ServerImpl::ClientData4Base::DoProcessIncomingMessages( - ClientMessageQueue& queue, size_t max) { + net::ClientMessageQueue& queue, size_t max) { DEBUG4("ProcessIncomingMessage()"); max = (std::min)(m_msgsBuf.size(), max); - std::span msgs = + std::span msgs = queue.ReadQueue(wpi::take_front(std::span{m_msgsBuf}, max)); // just map as a normal client into client=0 calls @@ -446,21 +444,21 @@ bool ServerImpl::ClientData4Base::DoProcessIncomingMessages( bool updatesub = false; for (const auto& elem : msgs) { // NOLINT // common case is value, so check that first - if (auto msg = std::get_if(&elem.contents)) { + if (auto msg = std::get_if(&elem.contents)) { ClientSetValue(msg->pubuid, msg->value); - } else if (auto msg = std::get_if(&elem.contents)) { + } else if (auto msg = std::get_if(&elem.contents)) { ClientPublish(msg->pubuid, msg->name, msg->typeStr, msg->properties, msg->options); updatepub = true; - } else if (auto msg = std::get_if(&elem.contents)) { + } else if (auto msg = std::get_if(&elem.contents)) { ClientUnpublish(msg->pubuid); updatepub = true; - } else if (auto msg = std::get_if(&elem.contents)) { + } else if (auto msg = std::get_if(&elem.contents)) { ClientSetProperties(msg->name, msg->update); - } else if (auto msg = std::get_if(&elem.contents)) { + } else if (auto msg = std::get_if(&elem.contents)) { ClientSubscribe(msg->subuid, msg->topicNames, msg->options); updatesub = true; - } else if (auto msg = std::get_if(&elem.contents)) { + } else if (auto msg = std::get_if(&elem.contents)) { ClientUnsubscribe(msg->subuid); updatesub = true; } @@ -502,7 +500,7 @@ bool ServerImpl::ClientData4::ProcessIncomingBinary( int pubuid; Value value; std::string error; - if (!WireDecodeBinary(&data, &pubuid, &value, &error, 0)) { + if (!net::WireDecodeBinary(&data, &pubuid, &value, &error, 0)) { m_wire.Disconnect(fmt::format("binary decode error: {}", error)); break; } @@ -512,7 +510,7 @@ bool ServerImpl::ClientData4::ProcessIncomingBinary( auto now = wpi::Now(); DEBUG4("RTT ping from {}, responding with time={}", m_id, now); m_wire.SendBinary( - [&](auto& os) { WireEncodeBinary(os, -1, now, value); }); + [&](auto& os) { net::WireEncodeBinary(os, -1, now, value); }); continue; } @@ -531,7 +529,7 @@ bool ServerImpl::ClientData4::ProcessIncomingBinary( } void ServerImpl::ClientData4::SendValue(TopicData* topic, const Value& value, - ValueSendMode mode) { + net::ValueSendMode mode) { m_outgoing.SendValue(topic->id, value, mode); } @@ -545,8 +543,8 @@ void ServerImpl::ClientData4::SendAnnounce(TopicData* topic, if (m_local) { int unsent = m_wire.WriteText([&](auto& os) { - WireEncodeAnnounce(os, topic->name, topic->id, topic->typeStr, - topic->properties, pubuid); + net::WireEncodeAnnounce(os, topic->name, topic->id, topic->typeStr, + topic->properties, pubuid); }); if (unsent < 0) { return; // error @@ -556,8 +554,8 @@ void ServerImpl::ClientData4::SendAnnounce(TopicData* topic, } } m_outgoing.SendMessage( - topic->id, AnnounceMsg{topic->name, static_cast(topic->id), - topic->typeStr, pubuid, topic->properties}); + topic->id, net::AnnounceMsg{topic->name, static_cast(topic->id), + topic->typeStr, pubuid, topic->properties}); m_server.m_controlReady = true; } @@ -569,8 +567,9 @@ void ServerImpl::ClientData4::SendUnannounce(TopicData* topic) { sent = false; if (m_local) { - int unsent = m_wire.WriteText( - [&](auto& os) { WireEncodeUnannounce(os, topic->name, topic->id); }); + int unsent = m_wire.WriteText([&](auto& os) { + net::WireEncodeUnannounce(os, topic->name, topic->id); + }); if (unsent < 0) { return; // error } @@ -579,7 +578,7 @@ void ServerImpl::ClientData4::SendUnannounce(TopicData* topic) { } } m_outgoing.SendMessage( - topic->id, UnannounceMsg{topic->name, static_cast(topic->id)}); + topic->id, net::UnannounceMsg{topic->name, static_cast(topic->id)}); m_outgoing.EraseId(topic->id); m_server.m_controlReady = true; } @@ -593,7 +592,7 @@ void ServerImpl::ClientData4::SendPropertiesUpdate(TopicData* topic, if (m_local) { int unsent = m_wire.WriteText([&](auto& os) { - WireEncodePropertiesUpdate(os, topic->name, update, ack); + net::WireEncodePropertiesUpdate(os, topic->name, update, ack); }); if (unsent < 0) { return; // error @@ -603,7 +602,7 @@ void ServerImpl::ClientData4::SendPropertiesUpdate(TopicData* topic, } } m_outgoing.SendMessage(topic->id, - PropertiesUpdateMsg{topic->name, update, ack}); + net::PropertiesUpdateMsg{topic->name, update, ack}); m_server.m_controlReady = true; } @@ -618,8 +617,8 @@ void ServerImpl::ClientData4::SendOutgoing(uint64_t curTimeMs, bool flush) { void ServerImpl::ClientData4::UpdatePeriod(TopicData::TopicClientData& tcd, TopicData* topic) { - uint32_t period = - CalculatePeriod(tcd.subscribers, [](auto& x) { return x->periodMs; }); + uint32_t period = net::CalculatePeriod(tcd.subscribers, + [](auto& x) { return x->periodMs; }); DEBUG4("updating {} period to {} ms", topic->name, period); m_outgoing.SetPeriod(topic->id, period); } @@ -640,21 +639,21 @@ bool ServerImpl::ClientData3::ProcessIncomingBinary( } void ServerImpl::ClientData3::SendValue(TopicData* topic, const Value& value, - ValueSendMode mode) { + net::ValueSendMode mode) { if (m_state != kStateRunning) { - if (mode == ValueSendMode::kImm) { - mode = ValueSendMode::kAll; + if (mode == net::ValueSendMode::kImm) { + mode = net::ValueSendMode::kAll; } } else if (m_local) { - mode = ValueSendMode::kImm; // always send local immediately + mode = net::ValueSendMode::kImm; // always send local immediately } TopicData3* topic3 = GetTopic3(topic); bool added = false; switch (mode) { - case ValueSendMode::kDisabled: // do nothing + case net::ValueSendMode::kDisabled: // do nothing break; - case ValueSendMode::kImm: // send immediately + case net::ValueSendMode::kImm: // send immediately ++topic3->seqNum; if (topic3->sentAssign) { net3::WireEncodeEntryUpdate(m_wire.Send().stream(), topic->id, @@ -669,7 +668,7 @@ void ServerImpl::ClientData3::SendValue(TopicData* topic, const Value& value, Flush(); } break; - case ValueSendMode::kNormal: { + case net::ValueSendMode::kNormal: { // replace, or append if not present wpi::DenseMap::iterator it; std::tie(it, added) = @@ -686,7 +685,7 @@ void ServerImpl::ClientData3::SendValue(TopicData* topic, const Value& value, } } // fallthrough - case ValueSendMode::kAll: // append to outgoing + case net::ValueSendMode::kAll: // append to outgoing if (!added) { m_outgoingValueMap[topic->id] = m_outgoing.size(); } @@ -889,7 +888,7 @@ void ServerImpl::ClientData3::ClientHello(std::string_view self_id, options.prefixMatch = true; sub = std::make_unique( this, std::span{{prefix}}, 0, options); - m_periodMs = UpdatePeriodCalc(m_periodMs, sub->periodMs); + m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->periodMs); m_setPeriodic(m_periodMs); { @@ -1212,7 +1211,7 @@ ServerImpl::ServerImpl(wpi::Logger& logger) : m_logger{logger} { std::pair ServerImpl::AddClient( std::string_view name, std::string_view connInfo, bool local, - WireConnection& wire, ServerImpl::SetPeriodicFunc setPeriodic) { + net::WireConnection& wire, ServerImpl::SetPeriodicFunc setPeriodic) { if (name.empty()) { name = "NT4"; } @@ -1842,7 +1841,7 @@ void ServerImpl::SetValue(ClientData* client, TopicData* topic, for (auto&& tcd : topic->clients) { if (tcd.first != client && - tcd.second.sendMode != ValueSendMode::kDisabled) { + tcd.second.sendMode != net::ValueSendMode::kDisabled) { tcd.first->SendValue(topic, value, tcd.second.sendMode); } } @@ -1942,8 +1941,8 @@ void ServerImpl::SendOutgoing(int clientId, uint64_t curTimeMs) { } } -void ServerImpl::SetLocal(ServerMessageHandler* local, - ClientMessageQueue* queue) { +void ServerImpl::SetLocal(net::ServerMessageHandler* local, + net::ClientMessageQueue* queue) { DEBUG4("SetLocal()"); m_local = local; m_localClient->SetQueue(queue); diff --git a/ntcore/src/main/native/cpp/net/ServerImpl.h b/ntcore/src/main/native/cpp/server/ServerImpl.h similarity index 90% rename from ntcore/src/main/native/cpp/net/ServerImpl.h rename to ntcore/src/main/native/cpp/server/ServerImpl.h index f4fae4113c7..55c39826178 100644 --- a/ntcore/src/main/native/cpp/net/ServerImpl.h +++ b/ntcore/src/main/native/cpp/server/ServerImpl.h @@ -21,14 +21,14 @@ #include #include -#include "ClientMessageQueue.h" -#include "Message.h" -#include "NetworkOutgoingQueue.h" -#include "NetworkPing.h" #include "PubSubOptions.h" -#include "WireConnection.h" -#include "WireDecoder.h" -#include "WireEncoder.h" +#include "net/ClientMessageQueue.h" +#include "net/Message.h" +#include "net/NetworkOutgoingQueue.h" +#include "net/NetworkPing.h" +#include "net/WireConnection.h" +#include "net/WireDecoder.h" +#include "net/WireEncoder.h" #include "net3/Message3.h" #include "net3/SequenceNumber.h" #include "net3/WireConnection3.h" @@ -41,15 +41,17 @@ class SmallVectorImpl; class raw_ostream; } // namespace wpi -namespace nt::net3 { -class WireConnection3; -} // namespace nt::net3 - namespace nt::net { - struct ClientMessage; class LocalInterface; class WireConnection; +} // namespace nt::net + +namespace nt::net3 { +class WireConnection3; +} // namespace nt::net3 + +namespace nt::server { class ServerImpl final { public: @@ -62,7 +64,8 @@ class ServerImpl final { void SendAllOutgoing(uint64_t curTimeMs, bool flush); void SendOutgoing(int clientId, uint64_t curTimeMs); - void SetLocal(ServerMessageHandler* local, ClientMessageQueue* queue); + void SetLocal(net::ServerMessageHandler* local, + net::ClientMessageQueue* queue); // these return true if any messages have been queued for later processing bool ProcessIncomingText(int clientId, std::string_view data); @@ -76,7 +79,7 @@ class ServerImpl final { // Caller must ensure WireConnection lifetime lasts until RemoveClient() call. std::pair AddClient(std::string_view name, std::string_view connInfo, bool local, - WireConnection& wire, + net::WireConnection& wire, SetPeriodicFunc setPeriodic); int AddClient3(std::string_view connInfo, bool local, net3::WireConnection3& wire, Connected3Func connected, @@ -149,15 +152,15 @@ class ServerImpl final { struct TopicClientData { wpi::SmallPtrSet publishers; wpi::SmallPtrSet subscribers; - ValueSendMode sendMode = ValueSendMode::kDisabled; + net::ValueSendMode sendMode = net::ValueSendMode::kDisabled; bool AddSubscriber(SubscriberData* sub) { bool added = subscribers.insert(sub).second; if (!sub->options.topicsOnly) { if (sub->options.sendAll) { - sendMode = ValueSendMode::kAll; - } else if (sendMode == ValueSendMode::kDisabled) { - sendMode = ValueSendMode::kNormal; + sendMode = net::ValueSendMode::kAll; + } else if (sendMode == net::ValueSendMode::kDisabled) { + sendMode = net::ValueSendMode::kNormal; } } return added; @@ -189,7 +192,7 @@ class ServerImpl final { virtual bool ProcessIncomingBinary(std::span data) = 0; virtual void SendValue(TopicData* topic, const Value& value, - ValueSendMode mode) = 0; + net::ValueSendMode mode) = 0; virtual void SendAnnounce(TopicData* topic, std::optional pubuid) = 0; virtual void SendUnannounce(TopicData* topic) = 0; virtual void SendPropertiesUpdate(TopicData* topic, const wpi::json& update, @@ -234,7 +237,8 @@ class ServerImpl final { TopicData* m_metaSub = nullptr; }; - class ClientData4Base : public ClientData, protected ClientMessageHandler { + class ClientData4Base : public ClientData, + protected net::ClientMessageHandler { public: ClientData4Base(std::string_view name, std::string_view connInfo, bool local, ServerImpl::SetPeriodicFunc setPeriodic, @@ -255,12 +259,12 @@ class ServerImpl final { void ClientSetValue(int pubuid, const Value& value) final; - bool DoProcessIncomingMessages(ClientMessageQueue& queue, size_t max); + bool DoProcessIncomingMessages(net::ClientMessageQueue& queue, size_t max); wpi::DenseMap m_announceSent; private: - std::array m_msgsBuf; + std::array m_msgsBuf; }; class ClientDataLocal final : public ClientData4Base { @@ -281,7 +285,7 @@ class ServerImpl final { } void SendValue(TopicData* topic, const Value& value, - ValueSendMode mode) final; + net::ValueSendMode mode) final; void SendAnnounce(TopicData* topic, std::optional pubuid) final; void SendUnannounce(TopicData* topic) final; void SendPropertiesUpdate(TopicData* topic, const wpi::json& update, @@ -289,17 +293,18 @@ class ServerImpl final { void SendOutgoing(uint64_t curTimeMs, bool flush) final {} void Flush() final {} - void SetQueue(ClientMessageQueue* queue) { m_queue = queue; } + void SetQueue(net::ClientMessageQueue* queue) { m_queue = queue; } private: - ClientMessageQueue* m_queue = nullptr; + net::ClientMessageQueue* m_queue = nullptr; }; class ClientData4 final : public ClientData4Base { public: ClientData4(std::string_view name, std::string_view connInfo, bool local, - WireConnection& wire, ServerImpl::SetPeriodicFunc setPeriodic, - ServerImpl& server, int id, wpi::Logger& logger) + net::WireConnection& wire, + ServerImpl::SetPeriodicFunc setPeriodic, ServerImpl& server, + int id, wpi::Logger& logger) : ClientData4Base{name, connInfo, local, setPeriodic, server, id, logger}, m_wire{wire}, @@ -319,7 +324,7 @@ class ServerImpl final { } void SendValue(TopicData* topic, const Value& value, - ValueSendMode mode) final; + net::ValueSendMode mode) final; void SendAnnounce(TopicData* topic, std::optional pubuid) final; void SendUnannounce(TopicData* topic) final; void SendPropertiesUpdate(TopicData* topic, const wpi::json& update, @@ -331,12 +336,12 @@ class ServerImpl final { void UpdatePeriod(TopicData::TopicClientData& tcd, TopicData* topic) final; public: - WireConnection& m_wire; + net::WireConnection& m_wire; private: - NetworkPing m_ping; - NetworkIncomingClientQueue m_incoming; - NetworkOutgoingQueue m_outgoing; + net::NetworkPing m_ping; + net::NetworkIncomingClientQueue m_incoming; + net::NetworkOutgoingQueue m_outgoing; }; class ClientData3 final : public ClientData, private net3::MessageHandler3 { @@ -358,7 +363,7 @@ class ServerImpl final { bool ProcessIncomingMessages(size_t max) final { return false; } void SendValue(TopicData* topic, const Value& value, - ValueSendMode mode) final; + net::ValueSendMode mode) final; void SendAnnounce(TopicData* topic, std::optional pubuid) final; void SendUnannounce(TopicData* topic) final; void SendPropertiesUpdate(TopicData* topic, const wpi::json& update, @@ -395,7 +400,7 @@ class ServerImpl final { State m_state{kStateInitial}; net3::WireDecoder3 m_decoder; - NetworkIncomingClientQueue m_incoming; + net::NetworkIncomingClientQueue m_incoming; std::vector m_outgoing; wpi::DenseMap m_outgoingValueMap; int64_t m_nextPubUid{1}; @@ -475,7 +480,7 @@ class ServerImpl final { }; wpi::Logger& m_logger; - ServerMessageHandler* m_local{nullptr}; + net::ServerMessageHandler* m_local{nullptr}; bool m_controlReady{false}; ClientDataLocal* m_localClient; @@ -510,4 +515,4 @@ class ServerImpl final { const wpi::json& update); }; -} // namespace nt::net +} // namespace nt::server diff --git a/ntcore/src/test/native/cpp/net/ServerImplTest.cpp b/ntcore/src/test/native/cpp/server/ServerImplTest.cpp similarity index 98% rename from ntcore/src/test/native/cpp/net/ServerImplTest.cpp rename to ntcore/src/test/native/cpp/server/ServerImplTest.cpp index dce0fb4a389..0db7f15d5b8 100644 --- a/ntcore/src/test/native/cpp/net/ServerImplTest.cpp +++ b/ntcore/src/test/native/cpp/server/ServerImplTest.cpp @@ -17,16 +17,16 @@ #include "../PubSubOptionsMatcher.h" #include "../TestPrinters.h" #include "../ValueMatcher.h" +#include "../net/MockClientMessageQueue.h" +#include "../net/MockMessageHandler.h" +#include "../net/MockWireConnection.h" #include "Handle.h" -#include "MockClientMessageQueue.h" -#include "MockMessageHandler.h" -#include "MockWireConnection.h" #include "gmock/gmock.h" #include "net/Message.h" -#include "net/ServerImpl.h" #include "net/WireEncoder.h" #include "ntcore_c.h" #include "ntcore_cpp.h" +#include "server/ServerImpl.h" using ::testing::_; using ::testing::AllOf; @@ -48,7 +48,7 @@ class ServerImplTest : public ::testing::Test { ::testing::StrictMock local; ::testing::StrictMock queue; wpi::MockLogger logger; - net::ServerImpl server{logger}; + server::ServerImpl server{logger}; }; TEST_F(ServerImplTest, AddClient) { From 7ba660ca3b8cb15b7db0694e627b043072054240 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Tue, 15 Oct 2024 23:06:43 -0700 Subject: [PATCH 05/26] [ntcore] Move ServerImpl detail classes to separate files --- ntcore/src/main/native/cpp/server/Constants.h | 13 + ntcore/src/main/native/cpp/server/Functions.h | 16 + .../native/cpp/server/MessagePackWriter.h | 31 + .../main/native/cpp/server/ServerClient.cpp | 63 + .../src/main/native/cpp/server/ServerClient.h | 98 ++ .../main/native/cpp/server/ServerClient3.cpp | 494 +++++++ .../main/native/cpp/server/ServerClient3.h | 97 ++ .../main/native/cpp/server/ServerClient4.cpp | 163 +++ .../main/native/cpp/server/ServerClient4.h | 60 + .../native/cpp/server/ServerClient4Base.cpp | 249 ++++ .../native/cpp/server/ServerClient4Base.h | 48 + .../native/cpp/server/ServerClientLocal.cpp | 52 + .../native/cpp/server/ServerClientLocal.h | 46 + .../src/main/native/cpp/server/ServerImpl.cpp | 1231 +---------------- .../src/main/native/cpp/server/ServerImpl.h | 456 +----- .../native/cpp/server/ServerPublisher.cpp | 47 + .../main/native/cpp/server/ServerPublisher.h | 31 + .../native/cpp/server/ServerSubscriber.cpp | 93 ++ .../main/native/cpp/server/ServerSubscriber.h | 66 + .../main/native/cpp/server/ServerTopic.cpp | 103 ++ .../src/main/native/cpp/server/ServerTopic.h | 102 ++ 21 files changed, 1941 insertions(+), 1618 deletions(-) create mode 100644 ntcore/src/main/native/cpp/server/Constants.h create mode 100644 ntcore/src/main/native/cpp/server/Functions.h create mode 100644 ntcore/src/main/native/cpp/server/MessagePackWriter.h create mode 100644 ntcore/src/main/native/cpp/server/ServerClient.cpp create mode 100644 ntcore/src/main/native/cpp/server/ServerClient.h create mode 100644 ntcore/src/main/native/cpp/server/ServerClient3.cpp create mode 100644 ntcore/src/main/native/cpp/server/ServerClient3.h create mode 100644 ntcore/src/main/native/cpp/server/ServerClient4.cpp create mode 100644 ntcore/src/main/native/cpp/server/ServerClient4.h create mode 100644 ntcore/src/main/native/cpp/server/ServerClient4Base.cpp create mode 100644 ntcore/src/main/native/cpp/server/ServerClient4Base.h create mode 100644 ntcore/src/main/native/cpp/server/ServerClientLocal.cpp create mode 100644 ntcore/src/main/native/cpp/server/ServerClientLocal.h create mode 100644 ntcore/src/main/native/cpp/server/ServerPublisher.cpp create mode 100644 ntcore/src/main/native/cpp/server/ServerPublisher.h create mode 100644 ntcore/src/main/native/cpp/server/ServerSubscriber.cpp create mode 100644 ntcore/src/main/native/cpp/server/ServerSubscriber.h create mode 100644 ntcore/src/main/native/cpp/server/ServerTopic.cpp create mode 100644 ntcore/src/main/native/cpp/server/ServerTopic.h diff --git a/ntcore/src/main/native/cpp/server/Constants.h b/ntcore/src/main/native/cpp/server/Constants.h new file mode 100644 index 00000000000..c04a7e08994 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/Constants.h @@ -0,0 +1,13 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +namespace nt::server { + +inline constexpr uint32_t kMinPeriodMs = 5; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/Functions.h b/ntcore/src/main/native/cpp/server/Functions.h new file mode 100644 index 00000000000..a1b4ac65db0 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/Functions.h @@ -0,0 +1,16 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +namespace nt::server { + +using SetPeriodicFunc = std::function; +using Connected3Func = + std::function; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/MessagePackWriter.h b/ntcore/src/main/native/cpp/server/MessagePackWriter.h new file mode 100644 index 00000000000..5ff99bc9c6f --- /dev/null +++ b/ntcore/src/main/native/cpp/server/MessagePackWriter.h @@ -0,0 +1,31 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include + +#include +#include + +namespace nt::server { + +struct Writer : public mpack::mpack_writer_t { + Writer() { + mpack::mpack_writer_init(this, buf, sizeof(buf)); + mpack::mpack_writer_set_context(this, &os); + mpack::mpack_writer_set_flush( + this, [](mpack::mpack_writer_t* w, const char* buffer, size_t count) { + static_cast(w->context)->write(buffer, count); + }); + } + + std::vector bytes; + wpi::raw_uvector_ostream os{bytes}; + char buf[128]; +}; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerClient.cpp b/ntcore/src/main/native/cpp/server/ServerClient.cpp new file mode 100644 index 00000000000..5f18149384e --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerClient.cpp @@ -0,0 +1,63 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "ServerClient.h" + +#include + +#include + +#include "server/MessagePackWriter.h" +#include "server/ServerImpl.h" +#include "server/ServerPublisher.h" + +using namespace nt::server; +using namespace mpack; + +void ServerClient::UpdateMetaClientPub() { + if (!m_metaPub) { + return; + } + Writer w; + mpack_start_array(&w, m_publishers.size()); + for (auto&& pub : m_publishers) { + mpack_write_object_bytes( + &w, reinterpret_cast(pub.second->metaClient.data()), + pub.second->metaClient.size()); + } + mpack_finish_array(&w); + if (mpack_writer_destroy(&w) == mpack_ok) { + m_server.SetValue(nullptr, m_metaPub, Value::MakeRaw(std::move(w.bytes))); + } +} + +void ServerClient::UpdateMetaClientSub() { + if (!m_metaSub) { + return; + } + Writer w; + mpack_start_array(&w, m_subscribers.size()); + for (auto&& sub : m_subscribers) { + mpack_write_object_bytes( + &w, reinterpret_cast(sub.second->metaClient.data()), + sub.second->metaClient.size()); + } + mpack_finish_array(&w); + if (mpack_writer_destroy(&w) == mpack_ok) { + m_server.SetValue(nullptr, m_metaSub, Value::MakeRaw(std::move(w.bytes))); + } +} + +std::span ServerClient::GetSubscribers( + std::string_view name, bool special, + wpi::SmallVectorImpl& buf) { + buf.resize(0); + for (auto&& subPair : m_subscribers) { + ServerSubscriber* subscriber = subPair.getSecond().get(); + if (subscriber->Matches(name, special)) { + buf.emplace_back(subscriber); + } + } + return {buf.data(), buf.size()}; +} diff --git a/ntcore/src/main/native/cpp/server/ServerClient.h b/ntcore/src/main/native/cpp/server/ServerClient.h new file mode 100644 index 00000000000..2c2f496a860 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerClient.h @@ -0,0 +1,98 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include "net/NetworkOutgoingQueue.h" +#include "server/Functions.h" +#include "server/ServerPublisher.h" +#include "server/ServerSubscriber.h" + +namespace wpi { +class Logger; +template +class SmallVectorImpl; +} // namespace wpi + +namespace nt::server { + +struct ServerTopic; +class ServerImpl; +struct TopicClientData; + +class ServerClient { + public: + ServerClient(std::string_view name, std::string_view connInfo, bool local, + SetPeriodicFunc setPeriodic, ServerImpl& server, int id, + wpi::Logger& logger) + : m_name{name}, + m_connInfo{connInfo}, + m_local{local}, + m_setPeriodic{std::move(setPeriodic)}, + m_server{server}, + m_id{id}, + m_logger{logger} {} + virtual ~ServerClient() = default; + + // these return true if any messages have been queued for later processing + virtual bool ProcessIncomingText(std::string_view data) = 0; + virtual bool ProcessIncomingBinary(std::span data) = 0; + + virtual void SendValue(ServerTopic* topic, const Value& value, + net::ValueSendMode mode) = 0; + virtual void SendAnnounce(ServerTopic* topic, std::optional pubuid) = 0; + virtual void SendUnannounce(ServerTopic* topic) = 0; + virtual void SendPropertiesUpdate(ServerTopic* topic, const wpi::json& update, + bool ack) = 0; + virtual void SendOutgoing(uint64_t curTimeMs, bool flush) = 0; + virtual void Flush() = 0; + + // later processing -- returns true if more to process + virtual bool ProcessIncomingMessages(size_t max) = 0; + + void UpdateMetaClientPub(); + void UpdateMetaClientSub(); + + std::span GetSubscribers( + std::string_view name, bool special, + wpi::SmallVectorImpl& buf); + + std::string_view GetName() const { return m_name; } + int GetId() const { return m_id; } + + virtual void UpdatePeriod(TopicClientData& tcd, ServerTopic* topic) {} + + protected: + std::string m_name; + std::string m_connInfo; + bool m_local; // local to machine + SetPeriodicFunc m_setPeriodic; + // TODO: make this per-topic? + uint32_t m_periodMs{UINT32_MAX}; + ServerImpl& m_server; + int m_id; + + wpi::Logger& m_logger; + + wpi::DenseMap> m_publishers; + wpi::DenseMap> m_subscribers; + + public: + // meta topics + ServerTopic* m_metaPub = nullptr; + ServerTopic* m_metaSub = nullptr; +}; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerClient3.cpp b/ntcore/src/main/native/cpp/server/ServerClient3.cpp new file mode 100644 index 00000000000..600e58f17bb --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerClient3.cpp @@ -0,0 +1,494 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "ServerClient3.h" + +#include +#include + +#include + +#include "Log.h" +#include "Types_internal.h" +#include "net3/WireEncoder3.h" +#include "server/ServerImpl.h" +#include "server/ServerPublisher.h" +#include "server/ServerTopic.h" + +using namespace nt::server; + +// maximum amount of time the wire can be not ready to send another +// transmission before we close the connection +static constexpr uint32_t kWireMaxNotReadyUs = 1000000; + +bool ServerClient3::TopicData3::UpdateFlags(ServerTopic* topic) { + unsigned int newFlags = topic->persistent ? NT_PERSISTENT : 0; + bool updated = flags != newFlags; + flags = newFlags; + return updated; +} + +bool ServerClient3::ProcessIncomingBinary(std::span data) { + if (!m_decoder.Execute(&data)) { + m_wire.Disconnect(m_decoder.GetError()); + } + return false; +} + +void ServerClient3::SendValue(ServerTopic* topic, const Value& value, + net::ValueSendMode mode) { + if (m_state != kStateRunning) { + if (mode == net::ValueSendMode::kImm) { + mode = net::ValueSendMode::kAll; + } + } else if (m_local) { + mode = net::ValueSendMode::kImm; // always send local immediately + } + TopicData3* topic3 = GetTopic3(topic); + bool added = false; + + switch (mode) { + case net::ValueSendMode::kDisabled: // do nothing + break; + case net::ValueSendMode::kImm: // send immediately + ++topic3->seqNum; + if (topic3->sentAssign) { + net3::WireEncodeEntryUpdate(m_wire.Send().stream(), topic->id, + topic3->seqNum.value(), value); + } else { + net3::WireEncodeEntryAssign(m_wire.Send().stream(), topic->name, + topic->id, topic3->seqNum.value(), value, + topic3->flags); + topic3->sentAssign = true; + } + if (m_local) { + Flush(); + } + break; + case net::ValueSendMode::kNormal: { + // replace, or append if not present + wpi::DenseMap::iterator it; + std::tie(it, added) = + m_outgoingValueMap.try_emplace(topic->id, m_outgoing.size()); + if (!added && it->second < m_outgoing.size()) { + auto& msg = m_outgoing[it->second]; + if (msg.Is(net3::Message3::kEntryUpdate) || + msg.Is(net3::Message3::kEntryAssign)) { + if (msg.id() == topic->id) { // should always be true + msg.SetValue(value); + break; + } + } + } + } + // fallthrough + case net::ValueSendMode::kAll: // append to outgoing + if (!added) { + m_outgoingValueMap[topic->id] = m_outgoing.size(); + } + ++topic3->seqNum; + if (topic3->sentAssign) { + m_outgoing.emplace_back(net3::Message3::EntryUpdate( + topic->id, topic3->seqNum.value(), value)); + } else { + m_outgoing.emplace_back(net3::Message3::EntryAssign( + topic->name, topic->id, topic3->seqNum.value(), value, + topic3->flags)); + topic3->sentAssign = true; + } + break; + } +} + +void ServerClient3::SendAnnounce(ServerTopic* topic, + std::optional pubuid) { + // ignore if we've not yet built the subscriber + if (m_subscribers.empty()) { + return; + } + + // subscribe to all non-special topics + if (!topic->special) { + topic->clients[this].AddSubscriber(m_subscribers[0].get()); + m_server.UpdateMetaTopicSub(topic); + } + + // NT3 requires a value to send the assign message, so the assign message + // will get sent when the first value is sent (by SendValue). +} + +void ServerClient3::SendUnannounce(ServerTopic* topic) { + auto it = m_topics3.find(topic); + if (it == m_topics3.end()) { + return; // never sent to client + } + bool sentAssign = it->second.sentAssign; + m_topics3.erase(it); + if (!sentAssign) { + return; // never sent to client + } + + // map to NT3 delete message + if (m_local && m_state == kStateRunning) { + net3::WireEncodeEntryDelete(m_wire.Send().stream(), topic->id); + Flush(); + } else { + m_outgoing.emplace_back(net3::Message3::EntryDelete(topic->id)); + } +} + +void ServerClient3::SendPropertiesUpdate(ServerTopic* topic, + const wpi::json& update, bool ack) { + if (ack) { + return; // we don't ack in NT3 + } + auto it = m_topics3.find(topic); + if (it == m_topics3.end()) { + return; // never sent to client + } + TopicData3* topic3 = &it->second; + // Don't send flags update unless we've already sent an assign message. + // The assign message will contain the updated flags when we eventually + // send it. + if (topic3->UpdateFlags(topic) && topic3->sentAssign) { + if (m_local && m_state == kStateRunning) { + net3::WireEncodeFlagsUpdate(m_wire.Send().stream(), topic->id, + topic3->flags); + Flush(); + } else { + m_outgoing.emplace_back( + net3::Message3::FlagsUpdate(topic->id, topic3->flags)); + } + } +} + +void ServerClient3::SendOutgoing(uint64_t curTimeMs, bool flush) { + if (m_outgoing.empty() || m_state != kStateRunning) { + return; // nothing to do + } + + // rate limit frequency of transmissions + if (curTimeMs < (m_lastSendMs + kMinPeriodMs)) { + return; + } + + if (!m_wire.Ready()) { + uint64_t lastFlushTime = m_wire.GetLastFlushTime(); + uint64_t now = wpi::Now(); + if (lastFlushTime != 0 && now > (lastFlushTime + kWireMaxNotReadyUs)) { + m_wire.Disconnect("transmit stalled"); + } + return; + } + + auto out = m_wire.Send(); + for (auto&& msg : m_outgoing) { + net3::WireEncode(out.stream(), msg); + } + m_wire.Flush(); + m_outgoing.resize(0); + m_outgoingValueMap.clear(); + m_lastSendMs = curTimeMs; +} + +void ServerClient3::KeepAlive() { + DEBUG4("KeepAlive({})", m_id); + if (m_state != kStateRunning) { + m_decoder.SetError("received unexpected KeepAlive message"); + return; + } + // ignore +} + +void ServerClient3::ServerHelloDone() { + DEBUG4("ServerHelloDone({})", m_id); + m_decoder.SetError("received unexpected ServerHelloDone message"); +} + +void ServerClient3::ClientHelloDone() { + DEBUG4("ClientHelloDone({})", m_id); + if (m_state != kStateServerHelloComplete) { + m_decoder.SetError("received unexpected ClientHelloDone message"); + return; + } + m_state = kStateRunning; +} + +void ServerClient3::ClearEntries() { + DEBUG4("ClearEntries({})", m_id); + if (m_state != kStateRunning) { + m_decoder.SetError("received unexpected ClearEntries message"); + return; + } + + for (auto topic3it : m_topics3) { + ServerTopic* topic = topic3it.first; + + // make sure we send assign the next time + topic3it.second.sentAssign = false; + + // unpublish from this client (if it was previously published) + if (topic3it.second.published) { + topic3it.second.published = false; + auto publisherIt = m_publishers.find(topic3it.second.pubuid); + if (publisherIt != m_publishers.end()) { + // remove publisher from topic + topic->RemovePublisher(this, publisherIt->second.get()); + + // remove publisher from client + m_publishers.erase(publisherIt); + + // update meta data + m_server.UpdateMetaTopicPub(topic); + UpdateMetaClientPub(); + } + } + + // set retained=false + m_server.SetProperties(this, topic, {{"retained", false}}); + } +} + +void ServerClient3::ProtoUnsup(unsigned int proto_rev) { + DEBUG4("ProtoUnsup({})", m_id); + m_decoder.SetError("received unexpected ProtoUnsup message"); +} + +void ServerClient3::ClientHello(std::string_view self_id, + unsigned int proto_rev) { + DEBUG4("ClientHello({}, '{}', {:04x})", m_id, self_id, proto_rev); + if (m_state != kStateInitial) { + m_decoder.SetError("received unexpected ClientHello message"); + return; + } + if (proto_rev != 0x0300) { + net3::WireEncodeProtoUnsup(m_wire.Send().stream(), 0x0300); + Flush(); + m_decoder.SetError( + fmt::format("unsupported protocol version {:04x}", proto_rev)); + return; + } + // create a unique name (just ignore provided client id) + m_name = fmt::format("NT3@{}", m_connInfo); + m_connected(m_name, 0x0300); + m_connected = nullptr; // no longer required + + // create client meta topics + m_metaPub = m_server.CreateMetaTopic(fmt::format("$clientpub${}", m_name)); + m_metaSub = m_server.CreateMetaTopic(fmt::format("$clientsub${}", m_name)); + + // subscribe and send initial assignments + auto& sub = m_subscribers[0]; + std::string prefix; + PubSubOptions options; + options.prefixMatch = true; + sub = std::make_unique( + this, std::span{{prefix}}, 0, options); + m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->periodMs); + m_setPeriodic(m_periodMs); + + { + auto out = m_wire.Send(); + net3::WireEncodeServerHello(out.stream(), 0, "server"); + for (auto&& topic : m_server.m_topics) { + if (topic && !topic->special && topic->IsPublished() && + topic->lastValue) { + DEBUG4("client {}: initial announce of '{}' (id {})", m_id, topic->name, + topic->id); + topic->clients[this].AddSubscriber(sub.get()); + m_server.UpdateMetaTopicSub(topic.get()); + + TopicData3* topic3 = GetTopic3(topic.get()); + ++topic3->seqNum; + net3::WireEncodeEntryAssign(out.stream(), topic->name, topic->id, + topic3->seqNum.value(), topic->lastValue, + topic3->flags); + topic3->sentAssign = true; + } + } + net3::WireEncodeServerHelloDone(out.stream()); + } + Flush(); + m_state = kStateServerHelloComplete; + + // update meta topics + UpdateMetaClientPub(); + UpdateMetaClientSub(); +} + +void ServerClient3::ServerHello(unsigned int flags, std::string_view self_id) { + DEBUG4("ServerHello({}, {}, {})", m_id, flags, self_id); + m_decoder.SetError("received unexpected ServerHello message"); +} + +void ServerClient3::EntryAssign(std::string_view name, unsigned int id, + unsigned int seq_num, const Value& value, + unsigned int flags) { + DEBUG4("EntryAssign({}, {}, {}, {}, {})", m_id, id, seq_num, + static_cast(value.type()), flags); + if (id != 0xffff) { + DEBUG3("ignored EntryAssign from {} with non-0xffff id {}", m_id, id); + return; + } + + // convert from NT3 info + auto typeStr = TypeToString(value.type()); + wpi::json properties = wpi::json::object(); + properties["retained"] = true; // treat all NT3 published topics as retained + properties["cached"] = true; // treat all NT3 published topics as cached + if ((flags & NT_PERSISTENT) != 0) { + properties["persistent"] = true; + } + + // create topic + auto topic = m_server.CreateTopic(this, name, typeStr, properties); + TopicData3* topic3 = GetTopic3(topic); + if (topic3->published || topic3->sentAssign) { + WARN("ignoring client {} duplicate publish of '{}'", m_id, name); + return; + } + ++topic3->seqNum; + topic3->published = true; + topic3->pubuid = m_nextPubUid++; + topic3->sentAssign = true; + + // create publisher + auto [publisherIt, isNew] = m_publishers.try_emplace( + topic3->pubuid, + std::make_unique(this, topic, topic3->pubuid)); + if (!isNew) { + return; // shouldn't happen, but just in case... + } + + // add publisher to topic + topic->AddPublisher(this, publisherIt->getSecond().get()); + + // update meta data + m_server.UpdateMetaTopicPub(topic); + UpdateMetaClientPub(); + + // acts as an announce + data update + SendAnnounce(topic, topic3->pubuid); + m_server.SetValue(this, topic, value); + + // respond with assign message with assigned topic ID + if (m_local && m_state == kStateRunning) { + net3::WireEncodeEntryAssign(m_wire.Send().stream(), topic->name, topic->id, + topic3->seqNum.value(), value, topic3->flags); + } else { + m_outgoing.emplace_back(net3::Message3::EntryAssign( + topic->name, topic->id, topic3->seqNum.value(), value, topic3->flags)); + } +} + +void ServerClient3::EntryUpdate(unsigned int id, unsigned int seq_num, + const Value& value) { + DEBUG4("EntryUpdate({}, {}, {}, {})", m_id, id, seq_num, + static_cast(value.type())); + if (m_state != kStateRunning) { + m_decoder.SetError("received unexpected EntryUpdate message"); + return; + } + + if (id >= m_server.m_topics.size()) { + DEBUG3("ignored EntryUpdate from {} on non-existent topic {}", m_id, id); + return; + } + ServerTopic* topic = m_server.m_topics[id].get(); + if (!topic || !topic->IsPublished()) { + DEBUG3("ignored EntryUpdate from {} on non-existent topic {}", m_id, id); + return; + } + + TopicData3* topic3 = GetTopic3(topic); + if (!topic3->published) { + topic3->published = true; + topic3->pubuid = m_nextPubUid++; + + // create publisher + auto [publisherIt, isNew] = m_publishers.try_emplace( + topic3->pubuid, + std::make_unique(this, topic, topic3->pubuid)); + if (isNew) { + // add publisher to topic + topic->AddPublisher(this, publisherIt->getSecond().get()); + + // update meta data + m_server.UpdateMetaTopicPub(topic); + UpdateMetaClientPub(); + } + } + topic3->seqNum = net3::SequenceNumber{seq_num}; + + m_server.SetValue(this, topic, value); +} + +void ServerClient3::FlagsUpdate(unsigned int id, unsigned int flags) { + DEBUG4("FlagsUpdate({}, {}, {})", m_id, id, flags); + if (m_state != kStateRunning) { + m_decoder.SetError("received unexpected FlagsUpdate message"); + return; + } + if (id >= m_server.m_topics.size()) { + DEBUG3("ignored FlagsUpdate from {} on non-existent topic {}", m_id, id); + return; + } + ServerTopic* topic = m_server.m_topics[id].get(); + if (!topic || !topic->IsPublished()) { + DEBUG3("ignored FlagsUpdate from {} on non-existent topic {}", m_id, id); + return; + } + if (topic->special) { + DEBUG3("ignored FlagsUpdate from {} on special topic {}", m_id, id); + return; + } + m_server.SetFlags(this, topic, flags); +} + +void ServerClient3::EntryDelete(unsigned int id) { + DEBUG4("EntryDelete({}, {})", m_id, id); + if (m_state != kStateRunning) { + m_decoder.SetError("received unexpected EntryDelete message"); + return; + } + if (id >= m_server.m_topics.size()) { + DEBUG3("ignored EntryDelete from {} on non-existent topic {}", m_id, id); + return; + } + ServerTopic* topic = m_server.m_topics[id].get(); + if (!topic || !topic->IsPublished()) { + DEBUG3("ignored EntryDelete from {} on non-existent topic {}", m_id, id); + return; + } + if (topic->special) { + DEBUG3("ignored EntryDelete from {} on special topic {}", m_id, id); + return; + } + + auto topic3it = m_topics3.find(topic); + if (topic3it != m_topics3.end()) { + // make sure we send assign the next time + topic3it->second.sentAssign = false; + + // unpublish from this client (if it was previously published) + if (topic3it->second.published) { + topic3it->second.published = false; + auto publisherIt = m_publishers.find(topic3it->second.pubuid); + if (publisherIt != m_publishers.end()) { + // remove publisher from topic + topic->RemovePublisher(this, publisherIt->second.get()); + + // remove publisher from client + m_publishers.erase(publisherIt); + + // update meta data + m_server.UpdateMetaTopicPub(topic); + UpdateMetaClientPub(); + } + } + } + + // set retained=false + m_server.SetProperties(this, topic, {{"retained", false}}); +} diff --git a/ntcore/src/main/native/cpp/server/ServerClient3.h b/ntcore/src/main/native/cpp/server/ServerClient3.h new file mode 100644 index 00000000000..7f61c059f3a --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerClient3.h @@ -0,0 +1,97 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +#include "ServerClient.h" +#include "net/ClientMessageQueue.h" +#include "net3/Message3.h" +#include "net3/SequenceNumber.h" +#include "net3/WireConnection3.h" +#include "net3/WireDecoder3.h" +#include "server/Functions.h" + +namespace nt::server { + +class ServerClient3 final : public ServerClient, private net3::MessageHandler3 { + public: + ServerClient3(std::string_view connInfo, bool local, + net3::WireConnection3& wire, Connected3Func connected, + SetPeriodicFunc setPeriodic, ServerImpl& server, int id, + wpi::Logger& logger) + : ServerClient{"", connInfo, local, setPeriodic, server, id, logger}, + m_connected{std::move(connected)}, + m_wire{wire}, + m_decoder{*this}, + m_incoming{logger} {} + + bool ProcessIncomingText(std::string_view data) final { return false; } + bool ProcessIncomingBinary(std::span data) final; + + bool ProcessIncomingMessages(size_t max) final { return false; } + + void SendValue(ServerTopic* topic, const Value& value, + net::ValueSendMode mode) final; + void SendAnnounce(ServerTopic* topic, std::optional pubuid) final; + void SendUnannounce(ServerTopic* topic) final; + void SendPropertiesUpdate(ServerTopic* topic, const wpi::json& update, + bool ack) final; + void SendOutgoing(uint64_t curTimeMs, bool flush) final; + + void Flush() final { m_wire.Flush(); } + + private: + // MessageHandler3 interface + void KeepAlive() final; + void ServerHelloDone() final; + void ClientHelloDone() final; + void ClearEntries() final; + void ProtoUnsup(unsigned int proto_rev) final; + void ClientHello(std::string_view self_id, unsigned int proto_rev) final; + void ServerHello(unsigned int flags, std::string_view self_id) final; + void EntryAssign(std::string_view name, unsigned int id, unsigned int seq_num, + const Value& value, unsigned int flags) final; + void EntryUpdate(unsigned int id, unsigned int seq_num, + const Value& value) final; + void FlagsUpdate(unsigned int id, unsigned int flags) final; + void EntryDelete(unsigned int id) final; + void ExecuteRpc(unsigned int id, unsigned int uid, + std::span params) final {} + void RpcResponse(unsigned int id, unsigned int uid, + std::span result) final {} + + Connected3Func m_connected; + net3::WireConnection3& m_wire; + + enum State { kStateInitial, kStateServerHelloComplete, kStateRunning }; + State m_state{kStateInitial}; + net3::WireDecoder3 m_decoder; + + net::NetworkIncomingClientQueue m_incoming; + std::vector m_outgoing; + wpi::DenseMap m_outgoingValueMap; + int64_t m_nextPubUid{1}; + uint64_t m_lastSendMs{0}; + + struct TopicData3 { + explicit TopicData3(ServerTopic* topic) { UpdateFlags(topic); } + + unsigned int flags{0}; + net3::SequenceNumber seqNum; + bool sentAssign{false}; + bool published{false}; + int64_t pubuid{0}; + + bool UpdateFlags(ServerTopic* topic); + }; + wpi::DenseMap m_topics3; + TopicData3* GetTopic3(ServerTopic* topic) { + return &m_topics3.try_emplace(topic, topic).first->second; + } +}; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerClient4.cpp b/ntcore/src/main/native/cpp/server/ServerClient4.cpp new file mode 100644 index 00000000000..5dd8c21c5e1 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerClient4.cpp @@ -0,0 +1,163 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "ServerClient4.h" + +#include + +#include + +#include "Log.h" +#include "net/WireDecoder.h" +#include "server/ServerImpl.h" +#include "server/ServerTopic.h" + +using namespace nt::server; + +bool ServerClient4::ProcessIncomingText(std::string_view data) { + constexpr int kMaxImmProcessing = 10; + bool queueWasEmpty = m_incoming.empty(); + // can't directly process, because we don't know how big it is + WireDecodeText(data, m_incoming, m_logger); + if (queueWasEmpty && + DoProcessIncomingMessages(m_incoming, kMaxImmProcessing)) { + m_wire.StopRead(); + return true; + } + return false; +} + +bool ServerClient4::ProcessIncomingBinary(std::span data) { + constexpr int kMaxImmProcessing = 10; + // if we've already queued, keep queuing + int count = m_incoming.empty() ? 0 : kMaxImmProcessing; + for (;;) { + if (data.empty()) { + break; + } + + // decode message + int pubuid; + Value value; + std::string error; + if (!net::WireDecodeBinary(&data, &pubuid, &value, &error, 0)) { + m_wire.Disconnect(fmt::format("binary decode error: {}", error)); + break; + } + + // respond to RTT ping + if (pubuid == -1) { + auto now = wpi::Now(); + DEBUG4("RTT ping from {}, responding with time={}", m_id, now); + m_wire.SendBinary( + [&](auto& os) { net::WireEncodeBinary(os, -1, now, value); }); + continue; + } + + // handle value set + if (++count < kMaxImmProcessing) { + ClientSetValue(pubuid, value); + } else { + m_incoming.ClientSetValue(pubuid, value); + } + } + if (count >= kMaxImmProcessing) { + m_wire.StopRead(); + return true; + } + return false; +} + +void ServerClient4::SendValue(ServerTopic* topic, const Value& value, + net::ValueSendMode mode) { + m_outgoing.SendValue(topic->id, value, mode); +} + +void ServerClient4::SendAnnounce(ServerTopic* topic, + std::optional pubuid) { + auto& sent = m_announceSent[topic]; + if (sent) { + return; + } + sent = true; + + if (m_local) { + int unsent = m_wire.WriteText([&](auto& os) { + net::WireEncodeAnnounce(os, topic->name, topic->id, topic->typeStr, + topic->properties, pubuid); + }); + if (unsent < 0) { + return; // error + } + if (unsent == 0 && m_wire.Flush() == 0) { + return; + } + } + m_outgoing.SendMessage( + topic->id, net::AnnounceMsg{topic->name, static_cast(topic->id), + topic->typeStr, pubuid, topic->properties}); + m_server.m_controlReady = true; +} + +void ServerClient4::SendUnannounce(ServerTopic* topic) { + auto& sent = m_announceSent[topic]; + if (!sent) { + return; + } + sent = false; + + if (m_local) { + int unsent = m_wire.WriteText([&](auto& os) { + net::WireEncodeUnannounce(os, topic->name, topic->id); + }); + if (unsent < 0) { + return; // error + } + if (unsent == 0 && m_wire.Flush() == 0) { + return; + } + } + m_outgoing.SendMessage( + topic->id, net::UnannounceMsg{topic->name, static_cast(topic->id)}); + m_outgoing.EraseId(topic->id); + m_server.m_controlReady = true; +} + +void ServerClient4::SendPropertiesUpdate(ServerTopic* topic, + const wpi::json& update, bool ack) { + if (!m_announceSent.lookup(topic)) { + return; + } + + if (m_local) { + int unsent = m_wire.WriteText([&](auto& os) { + net::WireEncodePropertiesUpdate(os, topic->name, update, ack); + }); + if (unsent < 0) { + return; // error + } + if (unsent == 0 && m_wire.Flush() == 0) { + return; + } + } + m_outgoing.SendMessage(topic->id, + net::PropertiesUpdateMsg{topic->name, update, ack}); + m_server.m_controlReady = true; +} + +void ServerClient4::SendOutgoing(uint64_t curTimeMs, bool flush) { + if (m_wire.GetVersion() >= 0x0401) { + if (!m_ping.Send(curTimeMs)) { + return; + } + } + m_outgoing.SendOutgoing(curTimeMs, flush); +} + +void ServerClient4::UpdatePeriod(TopicClientData& tcd, ServerTopic* topic) { + uint32_t period = net::CalculatePeriod(tcd.subscribers, + [](auto& x) { return x->periodMs; }); + DEBUG4("updating {} period to {} ms", topic->name, period); + m_outgoing.SetPeriod(topic->id, period); +} diff --git a/ntcore/src/main/native/cpp/server/ServerClient4.h b/ntcore/src/main/native/cpp/server/ServerClient4.h new file mode 100644 index 00000000000..ce3cb906f2f --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerClient4.h @@ -0,0 +1,60 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include "net/NetworkPing.h" +#include "net/WireConnection.h" +#include "server/Functions.h" +#include "server/ServerClient4Base.h" + +namespace nt::server { + +class ServerClient4 final : public ServerClient4Base { + public: + ServerClient4(std::string_view name, std::string_view connInfo, bool local, + net::WireConnection& wire, SetPeriodicFunc setPeriodic, + ServerImpl& server, int id, wpi::Logger& logger) + : ServerClient4Base{name, connInfo, local, setPeriodic, + server, id, logger}, + m_wire{wire}, + m_ping{wire}, + m_incoming{logger}, + m_outgoing{wire, local} {} + + bool ProcessIncomingText(std::string_view data) final; + bool ProcessIncomingBinary(std::span data) final; + + bool ProcessIncomingMessages(size_t max) final { + if (!DoProcessIncomingMessages(m_incoming, max)) { + m_wire.StartRead(); + return false; + } + return true; + } + + void SendValue(ServerTopic* topic, const Value& value, + net::ValueSendMode mode) final; + void SendAnnounce(ServerTopic* topic, std::optional pubuid) final; + void SendUnannounce(ServerTopic* topic) final; + void SendPropertiesUpdate(ServerTopic* topic, const wpi::json& update, + bool ack) final; + void SendOutgoing(uint64_t curTimeMs, bool flush) final; + + void Flush() final {} + + void UpdatePeriod(TopicClientData& tcd, ServerTopic* topic) final; + + public: + net::WireConnection& m_wire; + + private: + net::NetworkPing m_ping; + net::NetworkIncomingClientQueue m_incoming; + net::NetworkOutgoingQueue m_outgoing; +}; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp b/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp new file mode 100644 index 00000000000..5f8a894526e --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp @@ -0,0 +1,249 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "ServerClient4Base.h" + +#include +#include +#include + +#include +#include + +#include "Log.h" +#include "server/Constants.h" +#include "server/ServerImpl.h" +#include "server/ServerPublisher.h" + +using namespace nt::server; + +void ServerClient4Base::ClientPublish(int pubuid, std::string_view name, + std::string_view typeStr, + const wpi::json& properties, + const PubSubOptionsImpl& options) { + DEBUG3("ClientPublish({}, {}, {}, {})", m_id, name, pubuid, typeStr); + auto topic = m_server.CreateTopic(this, name, typeStr, properties); + + // create publisher + auto [publisherIt, isNew] = m_publishers.try_emplace( + pubuid, std::make_unique(this, topic, pubuid)); + if (!isNew) { + WARN("client {} duplicate publish of pubuid {}", m_id, pubuid); + } else { + // add publisher to topic + topic->AddPublisher(this, publisherIt->getSecond().get()); + + // update meta data + m_server.UpdateMetaTopicPub(topic); + } + + // respond with announce with pubuid to client + DEBUG4("client {}: announce {} pubuid {}", m_id, topic->name, pubuid); + SendAnnounce(topic, pubuid); +} + +void ServerClient4Base::ClientUnpublish(int pubuid) { + DEBUG3("ClientUnpublish({}, {})", m_id, pubuid); + auto publisherIt = m_publishers.find(pubuid); + if (publisherIt == m_publishers.end()) { + return; // nothing to do + } + auto publisher = publisherIt->getSecond().get(); + auto topic = publisher->topic; + + // remove publisher from topic + topic->RemovePublisher(this, publisher); + + // remove publisher from client + m_publishers.erase(publisherIt); + + // update meta data + m_server.UpdateMetaTopicPub(topic); + + // delete topic if no longer published + if (!topic->IsPublished()) { + m_server.DeleteTopic(topic); + } +} + +void ServerClient4Base::ClientSetProperties(std::string_view name, + const wpi::json& update) { + DEBUG4("ClientSetProperties({}, {}, {})", m_id, name, update.dump()); + auto topicIt = m_server.m_nameTopics.find(name); + if (topicIt == m_server.m_nameTopics.end() || + !topicIt->second->IsPublished()) { + WARN( + "server ignoring SetProperties({}) from client {} on unpublished topic " + "'{}'; publish or set a value first", + update.dump(), m_id, name); + return; // nothing to do + } + auto topic = topicIt->second; + if (topic->special) { + WARN("server ignoring SetProperties({}) from client {} on meta topic '{}'", + update.dump(), m_id, name); + return; // nothing to do + } + m_server.SetProperties(nullptr, topic, update); +} + +void ServerClient4Base::ClientSubscribe(int subuid, + std::span topicNames, + const PubSubOptionsImpl& options) { + DEBUG4("ClientSubscribe({}, ({}), {})", m_id, fmt::join(topicNames, ","), + subuid); + auto& sub = m_subscribers[subuid]; + bool replace = false; + if (sub) { + // replace subscription + sub->Update(topicNames, options); + replace = true; + } else { + // create + sub = std::make_unique(this, topicNames, subuid, options); + } + + // limit subscriber min period + if (sub->periodMs < kMinPeriodMs) { + sub->periodMs = kMinPeriodMs; + } + + // update periodic sender (if not local) + if (!m_local) { + m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->periodMs); + m_setPeriodic(m_periodMs); + } + + // see if this immediately subscribes to any topics + // for transmit efficiency, we want to batch announcements and values, so + // send announcements in first loop and remember what we want to send in + // second loop. + std::vector dataToSend; + dataToSend.reserve(m_server.m_topics.size()); + for (auto&& topic : m_server.m_topics) { + auto tcdIt = topic->clients.find(this); + bool removed = tcdIt != topic->clients.end() && replace && + tcdIt->second.subscribers.erase(sub.get()); + + // is client already subscribed? + bool wasSubscribed = + tcdIt != topic->clients.end() && !tcdIt->second.subscribers.empty(); + bool wasSubscribedValue = + wasSubscribed ? tcdIt->second.sendMode != net::ValueSendMode::kDisabled + : false; + + bool added = false; + if (sub->Matches(topic->name, topic->special)) { + if (tcdIt == topic->clients.end()) { + tcdIt = topic->clients.try_emplace(this).first; + } + tcdIt->second.AddSubscriber(sub.get()); + added = true; + } + + if (added ^ removed) { + UpdatePeriod(tcdIt->second, topic.get()); + m_server.UpdateMetaTopicSub(topic.get()); + } + + // announce topic to client if not previously announced + if (added && !removed && !wasSubscribed) { + DEBUG4("client {}: announce {}", m_id, topic->name); + SendAnnounce(topic.get(), std::nullopt); + } + + // send last value + if (added && !sub->options.topicsOnly && !wasSubscribedValue && + topic->lastValue) { + dataToSend.emplace_back(topic.get()); + } + } + + for (auto topic : dataToSend) { + DEBUG4("send last value for {} to client {}", topic->name, m_id); + SendValue(topic, topic->lastValue, net::ValueSendMode::kAll); + } +} + +void ServerClient4Base::ClientUnsubscribe(int subuid) { + DEBUG3("ClientUnsubscribe({}, {})", m_id, subuid); + auto subIt = m_subscribers.find(subuid); + if (subIt == m_subscribers.end() || !subIt->getSecond()) { + return; // nothing to do + } + auto sub = subIt->getSecond().get(); + + // remove from topics + for (auto&& topic : m_server.m_topics) { + auto tcdIt = topic->clients.find(this); + if (tcdIt != topic->clients.end()) { + if (tcdIt->second.subscribers.erase(sub)) { + UpdatePeriod(tcdIt->second, topic.get()); + m_server.UpdateMetaTopicSub(topic.get()); + } + } + } + + // delete it from client (future value sets will be ignored) + m_subscribers.erase(subIt); + + // loop over all subscribers to update period + if (!m_local) { + m_periodMs = net::CalculatePeriod( + m_subscribers, [](auto& x) { return x.getSecond()->periodMs; }); + m_setPeriodic(m_periodMs); + } +} + +void ServerClient4Base::ClientSetValue(int pubuid, const Value& value) { + DEBUG4("ClientSetValue({}, {})", m_id, pubuid); + auto publisherIt = m_publishers.find(pubuid); + if (publisherIt == m_publishers.end()) { + WARN("unrecognized client {} pubuid {}, ignoring set", m_id, pubuid); + return; // ignore unrecognized pubuids + } + auto topic = publisherIt->getSecond().get()->topic; + m_server.SetValue(this, topic, value); +} + +bool ServerClient4Base::DoProcessIncomingMessages( + net::ClientMessageQueue& queue, size_t max) { + DEBUG4("ProcessIncomingMessage()"); + max = (std::min)(m_msgsBuf.size(), max); + std::span msgs = + queue.ReadQueue(wpi::take_front(std::span{m_msgsBuf}, max)); + + // just map as a normal client into client=0 calls + bool updatepub = false; + bool updatesub = false; + for (const auto& elem : msgs) { // NOLINT + // common case is value, so check that first + if (auto msg = std::get_if(&elem.contents)) { + ClientSetValue(msg->pubuid, msg->value); + } else if (auto msg = std::get_if(&elem.contents)) { + ClientPublish(msg->pubuid, msg->name, msg->typeStr, msg->properties, + msg->options); + updatepub = true; + } else if (auto msg = std::get_if(&elem.contents)) { + ClientUnpublish(msg->pubuid); + updatepub = true; + } else if (auto msg = std::get_if(&elem.contents)) { + ClientSetProperties(msg->name, msg->update); + } else if (auto msg = std::get_if(&elem.contents)) { + ClientSubscribe(msg->subuid, msg->topicNames, msg->options); + updatesub = true; + } else if (auto msg = std::get_if(&elem.contents)) { + ClientUnsubscribe(msg->subuid); + updatesub = true; + } + } + if (updatepub) { + UpdateMetaClientPub(); + } + if (updatesub) { + UpdateMetaClientSub(); + } + + return msgs.size() == max; // don't know for sure, but there might be more +} diff --git a/ntcore/src/main/native/cpp/server/ServerClient4Base.h b/ntcore/src/main/native/cpp/server/ServerClient4Base.h new file mode 100644 index 00000000000..996fbb2a484 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerClient4Base.h @@ -0,0 +1,48 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +#include + +#include "net/ClientMessageQueue.h" +#include "server/Functions.h" +#include "server/ServerClient.h" + +namespace nt::server { + +class ServerClient4Base : public ServerClient, + protected net::ClientMessageHandler { + public: + ServerClient4Base(std::string_view name, std::string_view connInfo, + bool local, SetPeriodicFunc setPeriodic, ServerImpl& server, + int id, wpi::Logger& logger) + : ServerClient{name, connInfo, local, setPeriodic, server, id, logger} {} + + protected: + // ClientMessageHandler interface + void ClientPublish(int pubuid, std::string_view name, + std::string_view typeStr, const wpi::json& properties, + const PubSubOptionsImpl& options) final; + void ClientUnpublish(int pubuid) final; + void ClientSetProperties(std::string_view name, + const wpi::json& update) final; + void ClientSubscribe(int subuid, std::span topicNames, + const PubSubOptionsImpl& options) final; + void ClientUnsubscribe(int subuid) final; + + void ClientSetValue(int pubuid, const Value& value) final; + + bool DoProcessIncomingMessages(net::ClientMessageQueue& queue, size_t max); + + wpi::DenseMap m_announceSent; + + private: + std::array m_msgsBuf; +}; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp b/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp new file mode 100644 index 00000000000..557a30f11b3 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp @@ -0,0 +1,52 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "ServerClientLocal.h" + +#include "server/ServerImpl.h" + +using namespace nt::server; + +void ServerClientLocal::SendValue(ServerTopic* topic, const Value& value, + net::ValueSendMode mode) { + if (m_server.m_local) { + m_server.m_local->ServerSetValue(topic->localTopic, value); + } +} + +void ServerClientLocal::SendAnnounce(ServerTopic* topic, + std::optional pubuid) { + if (m_server.m_local) { + auto& sent = m_announceSent[topic]; + if (sent) { + return; + } + sent = true; + + topic->localTopic = m_server.m_local->ServerAnnounce( + topic->name, 0, topic->typeStr, topic->properties, pubuid); + } +} + +void ServerClientLocal::SendUnannounce(ServerTopic* topic) { + if (m_server.m_local) { + auto& sent = m_announceSent[topic]; + if (!sent) { + return; + } + sent = false; + m_server.m_local->ServerUnannounce(topic->name, topic->localTopic); + } +} + +void ServerClientLocal::SendPropertiesUpdate(ServerTopic* topic, + const wpi::json& update, + bool ack) { + if (m_server.m_local) { + if (!m_announceSent.lookup(topic)) { + return; + } + m_server.m_local->ServerPropertiesUpdate(topic->name, update, ack); + } +} diff --git a/ntcore/src/main/native/cpp/server/ServerClientLocal.h b/ntcore/src/main/native/cpp/server/ServerClientLocal.h new file mode 100644 index 00000000000..60895cf1b5d --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerClientLocal.h @@ -0,0 +1,46 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +#include "server/ServerClient4Base.h" + +namespace nt::server { + +class ServerClientLocal final : public ServerClient4Base { + public: + ServerClientLocal(ServerImpl& server, int id, wpi::Logger& logger) + : ServerClient4Base{"", "", true, [](uint32_t) {}, server, id, logger} {} + + bool ProcessIncomingText(std::string_view data) final { return false; } + bool ProcessIncomingBinary(std::span data) final { + return false; + } + + bool ProcessIncomingMessages(size_t max) final { + if (!m_queue) { + return false; + } + return DoProcessIncomingMessages(*m_queue, max); + } + + void SendValue(ServerTopic* topic, const Value& value, + net::ValueSendMode mode) final; + void SendAnnounce(ServerTopic* topic, std::optional pubuid) final; + void SendUnannounce(ServerTopic* topic) final; + void SendPropertiesUpdate(ServerTopic* topic, const wpi::json& update, + bool ack) final; + void SendOutgoing(uint64_t curTimeMs, bool flush) final {} + void Flush() final {} + + void SetQueue(net::ClientMessageQueue* queue) { m_queue = queue; } + + private: + net::ClientMessageQueue* m_queue = nullptr; +}; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.cpp b/ntcore/src/main/native/cpp/server/ServerImpl.cpp index 4d7c9fb5bca..63f48d5752b 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.cpp +++ b/ntcore/src/main/native/cpp/server/ServerImpl.cpp @@ -24,1194 +24,33 @@ #include #include "Log.h" -#include "Types_internal.h" -#include "net/Message.h" #include "net/WireEncoder.h" #include "net3/WireConnection3.h" #include "net3/WireEncoder3.h" #include "networktables/NetworkTableValue.h" #include "ntcore_c.h" +#include "server/MessagePackWriter.h" +#include "server/ServerClient3.h" +#include "server/ServerClient4.h" +#include "server/ServerClientLocal.h" +#include "server/ServerPublisher.h" +#include "server/ServerTopic.h" using namespace nt; using namespace nt::server; using namespace mpack; -// maximum amount of time the wire can be not ready to send another -// transmission before we close the connection -static constexpr uint32_t kWireMaxNotReadyUs = 1000000; - -namespace { -struct Writer : public mpack_writer_t { - Writer() { - mpack_writer_init(this, buf, sizeof(buf)); - mpack_writer_set_context(this, &os); - mpack_writer_set_flush( - this, [](mpack_writer_t* w, const char* buffer, size_t count) { - static_cast(w->context)->write(buffer, count); - }); - } - - std::vector bytes; - wpi::raw_uvector_ostream os{bytes}; - char buf[128]; -}; -} // namespace - -static void WriteOptions(mpack_writer_t& w, const PubSubOptionsImpl& options) { - int size = - (options.sendAll ? 1 : 0) + (options.topicsOnly ? 1 : 0) + - (options.periodicMs != PubSubOptionsImpl::kDefaultPeriodicMs ? 1 : 0) + - (options.prefixMatch ? 1 : 0); - mpack_start_map(&w, size); - if (options.sendAll) { - mpack_write_str(&w, "all"); - mpack_write_bool(&w, true); - } - if (options.topicsOnly) { - mpack_write_str(&w, "topicsonly"); - mpack_write_bool(&w, true); - } - if (options.periodicMs != PubSubOptionsImpl::kDefaultPeriodicMs) { - mpack_write_str(&w, "periodic"); - mpack_write_float(&w, options.periodicMs / 1000.0); - } - if (options.prefixMatch) { - mpack_write_str(&w, "prefix"); - mpack_write_bool(&w, true); - } - mpack_finish_map(&w); -} - -void ServerImpl::PublisherData::UpdateMeta() { - { - Writer w; - mpack_start_map(&w, 2); - mpack_write_str(&w, "uid"); - mpack_write_int(&w, pubuid); - mpack_write_str(&w, "topic"); - mpack_write_str(&w, topic->name); - mpack_finish_map(&w); - if (mpack_writer_destroy(&w) == mpack_ok) { - metaClient = std::move(w.bytes); - } - } - { - Writer w; - mpack_start_map(&w, 2); - mpack_write_str(&w, "client"); - if (client) { - mpack_write_str(&w, client->GetName()); - } else { - mpack_write_str(&w, ""); - } - mpack_write_str(&w, "pubuid"); - mpack_write_int(&w, pubuid); - mpack_finish_map(&w); - if (mpack_writer_destroy(&w) == mpack_ok) { - metaTopic = std::move(w.bytes); - } - } -} - -void ServerImpl::SubscriberData::UpdateMeta() { - { - Writer w; - mpack_start_map(&w, 3); - mpack_write_str(&w, "uid"); - mpack_write_int(&w, subuid); - mpack_write_str(&w, "topics"); - mpack_start_array(&w, topicNames.size()); - for (auto&& name : topicNames) { - mpack_write_str(&w, name); - } - mpack_finish_array(&w); - mpack_write_str(&w, "options"); - WriteOptions(w, options); - mpack_finish_map(&w); - if (mpack_writer_destroy(&w) == mpack_ok) { - metaClient = std::move(w.bytes); - } - } - { - Writer w; - mpack_start_map(&w, 3); - mpack_write_str(&w, "client"); - if (client) { - mpack_write_str(&w, client->GetName()); - } else { - mpack_write_str(&w, ""); - } - mpack_write_str(&w, "subuid"); - mpack_write_int(&w, subuid); - mpack_write_str(&w, "options"); - WriteOptions(w, options); - mpack_finish_map(&w); - if (mpack_writer_destroy(&w) == mpack_ok) { - metaTopic = std::move(w.bytes); - } - } -} - -void ServerImpl::ClientData::UpdateMetaClientPub() { - if (!m_metaPub) { - return; - } - Writer w; - mpack_start_array(&w, m_publishers.size()); - for (auto&& pub : m_publishers) { - mpack_write_object_bytes( - &w, reinterpret_cast(pub.second->metaClient.data()), - pub.second->metaClient.size()); - } - mpack_finish_array(&w); - if (mpack_writer_destroy(&w) == mpack_ok) { - m_server.SetValue(nullptr, m_metaPub, Value::MakeRaw(std::move(w.bytes))); - } -} - -void ServerImpl::ClientData::UpdateMetaClientSub() { - if (!m_metaSub) { - return; - } - Writer w; - mpack_start_array(&w, m_subscribers.size()); - for (auto&& sub : m_subscribers) { - mpack_write_object_bytes( - &w, reinterpret_cast(sub.second->metaClient.data()), - sub.second->metaClient.size()); - } - mpack_finish_array(&w); - if (mpack_writer_destroy(&w) == mpack_ok) { - m_server.SetValue(nullptr, m_metaSub, Value::MakeRaw(std::move(w.bytes))); - } -} - -std::span ServerImpl::ClientData::GetSubscribers( - std::string_view name, bool special, - wpi::SmallVectorImpl& buf) { - buf.resize(0); - for (auto&& subPair : m_subscribers) { - SubscriberData* subscriber = subPair.getSecond().get(); - if (subscriber->Matches(name, special)) { - buf.emplace_back(subscriber); - } - } - return {buf.data(), buf.size()}; -} - -void ServerImpl::ClientData4Base::ClientPublish( - int pubuid, std::string_view name, std::string_view typeStr, - const wpi::json& properties, const PubSubOptionsImpl& options) { - DEBUG3("ClientPublish({}, {}, {}, {})", m_id, name, pubuid, typeStr); - auto topic = m_server.CreateTopic(this, name, typeStr, properties); - - // create publisher - auto [publisherIt, isNew] = m_publishers.try_emplace( - pubuid, std::make_unique(this, topic, pubuid)); - if (!isNew) { - WARN("client {} duplicate publish of pubuid {}", m_id, pubuid); - } else { - // add publisher to topic - topic->AddPublisher(this, publisherIt->getSecond().get()); - - // update meta data - m_server.UpdateMetaTopicPub(topic); - } - - // respond with announce with pubuid to client - DEBUG4("client {}: announce {} pubuid {}", m_id, topic->name, pubuid); - SendAnnounce(topic, pubuid); -} - -void ServerImpl::ClientData4Base::ClientUnpublish(int pubuid) { - DEBUG3("ClientUnpublish({}, {})", m_id, pubuid); - auto publisherIt = m_publishers.find(pubuid); - if (publisherIt == m_publishers.end()) { - return; // nothing to do - } - auto publisher = publisherIt->getSecond().get(); - auto topic = publisher->topic; - - // remove publisher from topic - topic->RemovePublisher(this, publisher); - - // remove publisher from client - m_publishers.erase(publisherIt); - - // update meta data - m_server.UpdateMetaTopicPub(topic); - - // delete topic if no longer published - if (!topic->IsPublished()) { - m_server.DeleteTopic(topic); - } -} - -void ServerImpl::ClientData4Base::ClientSetProperties(std::string_view name, - const wpi::json& update) { - DEBUG4("ClientSetProperties({}, {}, {})", m_id, name, update.dump()); - auto topicIt = m_server.m_nameTopics.find(name); - if (topicIt == m_server.m_nameTopics.end() || - !topicIt->second->IsPublished()) { - WARN( - "server ignoring SetProperties({}) from client {} on unpublished topic " - "'{}'; publish or set a value first", - update.dump(), m_id, name); - return; // nothing to do - } - auto topic = topicIt->second; - if (topic->special) { - WARN("server ignoring SetProperties({}) from client {} on meta topic '{}'", - update.dump(), m_id, name); - return; // nothing to do - } - m_server.SetProperties(nullptr, topic, update); -} - -void ServerImpl::ClientData4Base::ClientSubscribe( - int subuid, std::span topicNames, - const PubSubOptionsImpl& options) { - DEBUG4("ClientSubscribe({}, ({}), {})", m_id, fmt::join(topicNames, ","), - subuid); - auto& sub = m_subscribers[subuid]; - bool replace = false; - if (sub) { - // replace subscription - sub->Update(topicNames, options); - replace = true; - } else { - // create - sub = std::make_unique(this, topicNames, subuid, options); - } - - // limit subscriber min period - if (sub->periodMs < kMinPeriodMs) { - sub->periodMs = kMinPeriodMs; - } - - // update periodic sender (if not local) - if (!m_local) { - m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->periodMs); - m_setPeriodic(m_periodMs); - } - - // see if this immediately subscribes to any topics - // for transmit efficiency, we want to batch announcements and values, so - // send announcements in first loop and remember what we want to send in - // second loop. - std::vector dataToSend; - dataToSend.reserve(m_server.m_topics.size()); - for (auto&& topic : m_server.m_topics) { - auto tcdIt = topic->clients.find(this); - bool removed = tcdIt != topic->clients.end() && replace && - tcdIt->second.subscribers.erase(sub.get()); - - // is client already subscribed? - bool wasSubscribed = - tcdIt != topic->clients.end() && !tcdIt->second.subscribers.empty(); - bool wasSubscribedValue = - wasSubscribed ? tcdIt->second.sendMode != net::ValueSendMode::kDisabled - : false; - - bool added = false; - if (sub->Matches(topic->name, topic->special)) { - if (tcdIt == topic->clients.end()) { - tcdIt = topic->clients.try_emplace(this).first; - } - tcdIt->second.AddSubscriber(sub.get()); - added = true; - } - - if (added ^ removed) { - UpdatePeriod(tcdIt->second, topic.get()); - m_server.UpdateMetaTopicSub(topic.get()); - } - - // announce topic to client if not previously announced - if (added && !removed && !wasSubscribed) { - DEBUG4("client {}: announce {}", m_id, topic->name); - SendAnnounce(topic.get(), std::nullopt); - } - - // send last value - if (added && !sub->options.topicsOnly && !wasSubscribedValue && - topic->lastValue) { - dataToSend.emplace_back(topic.get()); - } - } - - for (auto topic : dataToSend) { - DEBUG4("send last value for {} to client {}", topic->name, m_id); - SendValue(topic, topic->lastValue, net::ValueSendMode::kAll); - } -} - -void ServerImpl::ClientData4Base::ClientUnsubscribe(int subuid) { - DEBUG3("ClientUnsubscribe({}, {})", m_id, subuid); - auto subIt = m_subscribers.find(subuid); - if (subIt == m_subscribers.end() || !subIt->getSecond()) { - return; // nothing to do - } - auto sub = subIt->getSecond().get(); - - // remove from topics - for (auto&& topic : m_server.m_topics) { - auto tcdIt = topic->clients.find(this); - if (tcdIt != topic->clients.end()) { - if (tcdIt->second.subscribers.erase(sub)) { - UpdatePeriod(tcdIt->second, topic.get()); - m_server.UpdateMetaTopicSub(topic.get()); - } - } - } - - // delete it from client (future value sets will be ignored) - m_subscribers.erase(subIt); - - // loop over all subscribers to update period - if (!m_local) { - m_periodMs = net::CalculatePeriod( - m_subscribers, [](auto& x) { return x.getSecond()->periodMs; }); - m_setPeriodic(m_periodMs); - } -} - -void ServerImpl::ClientData4Base::ClientSetValue(int pubuid, - const Value& value) { - DEBUG4("ClientSetValue({}, {})", m_id, pubuid); - auto publisherIt = m_publishers.find(pubuid); - if (publisherIt == m_publishers.end()) { - WARN("unrecognized client {} pubuid {}, ignoring set", m_id, pubuid); - return; // ignore unrecognized pubuids - } - auto topic = publisherIt->getSecond().get()->topic; - m_server.SetValue(this, topic, value); -} - -void ServerImpl::ClientDataLocal::SendValue(TopicData* topic, - const Value& value, - net::ValueSendMode mode) { - if (m_server.m_local) { - m_server.m_local->ServerSetValue(topic->localTopic, value); - } -} - -void ServerImpl::ClientDataLocal::SendAnnounce(TopicData* topic, - std::optional pubuid) { - if (m_server.m_local) { - auto& sent = m_announceSent[topic]; - if (sent) { - return; - } - sent = true; - - topic->localTopic = m_server.m_local->ServerAnnounce( - topic->name, 0, topic->typeStr, topic->properties, pubuid); - } -} - -void ServerImpl::ClientDataLocal::SendUnannounce(TopicData* topic) { - if (m_server.m_local) { - auto& sent = m_announceSent[topic]; - if (!sent) { - return; - } - sent = false; - m_server.m_local->ServerUnannounce(topic->name, topic->localTopic); - } -} - -void ServerImpl::ClientDataLocal::SendPropertiesUpdate(TopicData* topic, - const wpi::json& update, - bool ack) { - if (m_server.m_local) { - if (!m_announceSent.lookup(topic)) { - return; - } - m_server.m_local->ServerPropertiesUpdate(topic->name, update, ack); - } -} - -bool ServerImpl::ClientData4Base::DoProcessIncomingMessages( - net::ClientMessageQueue& queue, size_t max) { - DEBUG4("ProcessIncomingMessage()"); - max = (std::min)(m_msgsBuf.size(), max); - std::span msgs = - queue.ReadQueue(wpi::take_front(std::span{m_msgsBuf}, max)); - - // just map as a normal client into client=0 calls - bool updatepub = false; - bool updatesub = false; - for (const auto& elem : msgs) { // NOLINT - // common case is value, so check that first - if (auto msg = std::get_if(&elem.contents)) { - ClientSetValue(msg->pubuid, msg->value); - } else if (auto msg = std::get_if(&elem.contents)) { - ClientPublish(msg->pubuid, msg->name, msg->typeStr, msg->properties, - msg->options); - updatepub = true; - } else if (auto msg = std::get_if(&elem.contents)) { - ClientUnpublish(msg->pubuid); - updatepub = true; - } else if (auto msg = std::get_if(&elem.contents)) { - ClientSetProperties(msg->name, msg->update); - } else if (auto msg = std::get_if(&elem.contents)) { - ClientSubscribe(msg->subuid, msg->topicNames, msg->options); - updatesub = true; - } else if (auto msg = std::get_if(&elem.contents)) { - ClientUnsubscribe(msg->subuid); - updatesub = true; - } - } - if (updatepub) { - UpdateMetaClientPub(); - } - if (updatesub) { - UpdateMetaClientSub(); - } - - return msgs.size() == max; // don't know for sure, but there might be more -} - -bool ServerImpl::ClientData4::ProcessIncomingText(std::string_view data) { - constexpr int kMaxImmProcessing = 10; - bool queueWasEmpty = m_incoming.empty(); - // can't directly process, because we don't know how big it is - WireDecodeText(data, m_incoming, m_logger); - if (queueWasEmpty && - DoProcessIncomingMessages(m_incoming, kMaxImmProcessing)) { - m_wire.StopRead(); - return true; - } - return false; -} - -bool ServerImpl::ClientData4::ProcessIncomingBinary( - std::span data) { - constexpr int kMaxImmProcessing = 10; - // if we've already queued, keep queuing - int count = m_incoming.empty() ? 0 : kMaxImmProcessing; - for (;;) { - if (data.empty()) { - break; - } - - // decode message - int pubuid; - Value value; - std::string error; - if (!net::WireDecodeBinary(&data, &pubuid, &value, &error, 0)) { - m_wire.Disconnect(fmt::format("binary decode error: {}", error)); - break; - } - - // respond to RTT ping - if (pubuid == -1) { - auto now = wpi::Now(); - DEBUG4("RTT ping from {}, responding with time={}", m_id, now); - m_wire.SendBinary( - [&](auto& os) { net::WireEncodeBinary(os, -1, now, value); }); - continue; - } - - // handle value set - if (++count < kMaxImmProcessing) { - ClientSetValue(pubuid, value); - } else { - m_incoming.ClientSetValue(pubuid, value); - } - } - if (count >= kMaxImmProcessing) { - m_wire.StopRead(); - return true; - } - return false; -} - -void ServerImpl::ClientData4::SendValue(TopicData* topic, const Value& value, - net::ValueSendMode mode) { - m_outgoing.SendValue(topic->id, value, mode); -} - -void ServerImpl::ClientData4::SendAnnounce(TopicData* topic, - std::optional pubuid) { - auto& sent = m_announceSent[topic]; - if (sent) { - return; - } - sent = true; - - if (m_local) { - int unsent = m_wire.WriteText([&](auto& os) { - net::WireEncodeAnnounce(os, topic->name, topic->id, topic->typeStr, - topic->properties, pubuid); - }); - if (unsent < 0) { - return; // error - } - if (unsent == 0 && m_wire.Flush() == 0) { - return; - } - } - m_outgoing.SendMessage( - topic->id, net::AnnounceMsg{topic->name, static_cast(topic->id), - topic->typeStr, pubuid, topic->properties}); - m_server.m_controlReady = true; -} - -void ServerImpl::ClientData4::SendUnannounce(TopicData* topic) { - auto& sent = m_announceSent[topic]; - if (!sent) { - return; - } - sent = false; - - if (m_local) { - int unsent = m_wire.WriteText([&](auto& os) { - net::WireEncodeUnannounce(os, topic->name, topic->id); - }); - if (unsent < 0) { - return; // error - } - if (unsent == 0 && m_wire.Flush() == 0) { - return; - } - } - m_outgoing.SendMessage( - topic->id, net::UnannounceMsg{topic->name, static_cast(topic->id)}); - m_outgoing.EraseId(topic->id); - m_server.m_controlReady = true; -} - -void ServerImpl::ClientData4::SendPropertiesUpdate(TopicData* topic, - const wpi::json& update, - bool ack) { - if (!m_announceSent.lookup(topic)) { - return; - } - - if (m_local) { - int unsent = m_wire.WriteText([&](auto& os) { - net::WireEncodePropertiesUpdate(os, topic->name, update, ack); - }); - if (unsent < 0) { - return; // error - } - if (unsent == 0 && m_wire.Flush() == 0) { - return; - } - } - m_outgoing.SendMessage(topic->id, - net::PropertiesUpdateMsg{topic->name, update, ack}); - m_server.m_controlReady = true; -} - -void ServerImpl::ClientData4::SendOutgoing(uint64_t curTimeMs, bool flush) { - if (m_wire.GetVersion() >= 0x0401) { - if (!m_ping.Send(curTimeMs)) { - return; - } - } - m_outgoing.SendOutgoing(curTimeMs, flush); -} - -void ServerImpl::ClientData4::UpdatePeriod(TopicData::TopicClientData& tcd, - TopicData* topic) { - uint32_t period = net::CalculatePeriod(tcd.subscribers, - [](auto& x) { return x->periodMs; }); - DEBUG4("updating {} period to {} ms", topic->name, period); - m_outgoing.SetPeriod(topic->id, period); -} - -bool ServerImpl::ClientData3::TopicData3::UpdateFlags(TopicData* topic) { - unsigned int newFlags = topic->persistent ? NT_PERSISTENT : 0; - bool updated = flags != newFlags; - flags = newFlags; - return updated; -} - -bool ServerImpl::ClientData3::ProcessIncomingBinary( - std::span data) { - if (!m_decoder.Execute(&data)) { - m_wire.Disconnect(m_decoder.GetError()); - } - return false; -} - -void ServerImpl::ClientData3::SendValue(TopicData* topic, const Value& value, - net::ValueSendMode mode) { - if (m_state != kStateRunning) { - if (mode == net::ValueSendMode::kImm) { - mode = net::ValueSendMode::kAll; - } - } else if (m_local) { - mode = net::ValueSendMode::kImm; // always send local immediately - } - TopicData3* topic3 = GetTopic3(topic); - bool added = false; - - switch (mode) { - case net::ValueSendMode::kDisabled: // do nothing - break; - case net::ValueSendMode::kImm: // send immediately - ++topic3->seqNum; - if (topic3->sentAssign) { - net3::WireEncodeEntryUpdate(m_wire.Send().stream(), topic->id, - topic3->seqNum.value(), value); - } else { - net3::WireEncodeEntryAssign(m_wire.Send().stream(), topic->name, - topic->id, topic3->seqNum.value(), value, - topic3->flags); - topic3->sentAssign = true; - } - if (m_local) { - Flush(); - } - break; - case net::ValueSendMode::kNormal: { - // replace, or append if not present - wpi::DenseMap::iterator it; - std::tie(it, added) = - m_outgoingValueMap.try_emplace(topic->id, m_outgoing.size()); - if (!added && it->second < m_outgoing.size()) { - auto& msg = m_outgoing[it->second]; - if (msg.Is(net3::Message3::kEntryUpdate) || - msg.Is(net3::Message3::kEntryAssign)) { - if (msg.id() == topic->id) { // should always be true - msg.SetValue(value); - break; - } - } - } - } - // fallthrough - case net::ValueSendMode::kAll: // append to outgoing - if (!added) { - m_outgoingValueMap[topic->id] = m_outgoing.size(); - } - ++topic3->seqNum; - if (topic3->sentAssign) { - m_outgoing.emplace_back(net3::Message3::EntryUpdate( - topic->id, topic3->seqNum.value(), value)); - } else { - m_outgoing.emplace_back(net3::Message3::EntryAssign( - topic->name, topic->id, topic3->seqNum.value(), value, - topic3->flags)); - topic3->sentAssign = true; - } - break; - } -} - -void ServerImpl::ClientData3::SendAnnounce(TopicData* topic, - std::optional pubuid) { - // ignore if we've not yet built the subscriber - if (m_subscribers.empty()) { - return; - } - - // subscribe to all non-special topics - if (!topic->special) { - topic->clients[this].AddSubscriber(m_subscribers[0].get()); - m_server.UpdateMetaTopicSub(topic); - } - - // NT3 requires a value to send the assign message, so the assign message - // will get sent when the first value is sent (by SendValue). -} - -void ServerImpl::ClientData3::SendUnannounce(TopicData* topic) { - auto it = m_topics3.find(topic); - if (it == m_topics3.end()) { - return; // never sent to client - } - bool sentAssign = it->second.sentAssign; - m_topics3.erase(it); - if (!sentAssign) { - return; // never sent to client - } - - // map to NT3 delete message - if (m_local && m_state == kStateRunning) { - net3::WireEncodeEntryDelete(m_wire.Send().stream(), topic->id); - Flush(); - } else { - m_outgoing.emplace_back(net3::Message3::EntryDelete(topic->id)); - } -} - -void ServerImpl::ClientData3::SendPropertiesUpdate(TopicData* topic, - const wpi::json& update, - bool ack) { - if (ack) { - return; // we don't ack in NT3 - } - auto it = m_topics3.find(topic); - if (it == m_topics3.end()) { - return; // never sent to client - } - TopicData3* topic3 = &it->second; - // Don't send flags update unless we've already sent an assign message. - // The assign message will contain the updated flags when we eventually - // send it. - if (topic3->UpdateFlags(topic) && topic3->sentAssign) { - if (m_local && m_state == kStateRunning) { - net3::WireEncodeFlagsUpdate(m_wire.Send().stream(), topic->id, - topic3->flags); - Flush(); - } else { - m_outgoing.emplace_back( - net3::Message3::FlagsUpdate(topic->id, topic3->flags)); - } - } -} - -void ServerImpl::ClientData3::SendOutgoing(uint64_t curTimeMs, bool flush) { - if (m_outgoing.empty() || m_state != kStateRunning) { - return; // nothing to do - } - - // rate limit frequency of transmissions - if (curTimeMs < (m_lastSendMs + kMinPeriodMs)) { - return; - } - - if (!m_wire.Ready()) { - uint64_t lastFlushTime = m_wire.GetLastFlushTime(); - uint64_t now = wpi::Now(); - if (lastFlushTime != 0 && now > (lastFlushTime + kWireMaxNotReadyUs)) { - m_wire.Disconnect("transmit stalled"); - } - return; - } - - auto out = m_wire.Send(); - for (auto&& msg : m_outgoing) { - net3::WireEncode(out.stream(), msg); - } - m_wire.Flush(); - m_outgoing.resize(0); - m_outgoingValueMap.clear(); - m_lastSendMs = curTimeMs; -} - -void ServerImpl::ClientData3::KeepAlive() { - DEBUG4("KeepAlive({})", m_id); - if (m_state != kStateRunning) { - m_decoder.SetError("received unexpected KeepAlive message"); - return; - } - // ignore -} - -void ServerImpl::ClientData3::ServerHelloDone() { - DEBUG4("ServerHelloDone({})", m_id); - m_decoder.SetError("received unexpected ServerHelloDone message"); -} - -void ServerImpl::ClientData3::ClientHelloDone() { - DEBUG4("ClientHelloDone({})", m_id); - if (m_state != kStateServerHelloComplete) { - m_decoder.SetError("received unexpected ClientHelloDone message"); - return; - } - m_state = kStateRunning; -} - -void ServerImpl::ClientData3::ClearEntries() { - DEBUG4("ClearEntries({})", m_id); - if (m_state != kStateRunning) { - m_decoder.SetError("received unexpected ClearEntries message"); - return; - } - - for (auto topic3it : m_topics3) { - TopicData* topic = topic3it.first; - - // make sure we send assign the next time - topic3it.second.sentAssign = false; - - // unpublish from this client (if it was previously published) - if (topic3it.second.published) { - topic3it.second.published = false; - auto publisherIt = m_publishers.find(topic3it.second.pubuid); - if (publisherIt != m_publishers.end()) { - // remove publisher from topic - topic->RemovePublisher(this, publisherIt->second.get()); - - // remove publisher from client - m_publishers.erase(publisherIt); - - // update meta data - m_server.UpdateMetaTopicPub(topic); - UpdateMetaClientPub(); - } - } - - // set retained=false - m_server.SetProperties(this, topic, {{"retained", false}}); - } -} - -void ServerImpl::ClientData3::ProtoUnsup(unsigned int proto_rev) { - DEBUG4("ProtoUnsup({})", m_id); - m_decoder.SetError("received unexpected ProtoUnsup message"); -} - -void ServerImpl::ClientData3::ClientHello(std::string_view self_id, - unsigned int proto_rev) { - DEBUG4("ClientHello({}, '{}', {:04x})", m_id, self_id, proto_rev); - if (m_state != kStateInitial) { - m_decoder.SetError("received unexpected ClientHello message"); - return; - } - if (proto_rev != 0x0300) { - net3::WireEncodeProtoUnsup(m_wire.Send().stream(), 0x0300); - Flush(); - m_decoder.SetError( - fmt::format("unsupported protocol version {:04x}", proto_rev)); - return; - } - // create a unique name (just ignore provided client id) - m_name = fmt::format("NT3@{}", m_connInfo); - m_connected(m_name, 0x0300); - m_connected = nullptr; // no longer required - - // create client meta topics - m_metaPub = m_server.CreateMetaTopic(fmt::format("$clientpub${}", m_name)); - m_metaSub = m_server.CreateMetaTopic(fmt::format("$clientsub${}", m_name)); - - // subscribe and send initial assignments - auto& sub = m_subscribers[0]; - std::string prefix; - PubSubOptions options; - options.prefixMatch = true; - sub = std::make_unique( - this, std::span{{prefix}}, 0, options); - m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->periodMs); - m_setPeriodic(m_periodMs); - - { - auto out = m_wire.Send(); - net3::WireEncodeServerHello(out.stream(), 0, "server"); - for (auto&& topic : m_server.m_topics) { - if (topic && !topic->special && topic->IsPublished() && - topic->lastValue) { - DEBUG4("client {}: initial announce of '{}' (id {})", m_id, topic->name, - topic->id); - topic->clients[this].AddSubscriber(sub.get()); - m_server.UpdateMetaTopicSub(topic.get()); - - TopicData3* topic3 = GetTopic3(topic.get()); - ++topic3->seqNum; - net3::WireEncodeEntryAssign(out.stream(), topic->name, topic->id, - topic3->seqNum.value(), topic->lastValue, - topic3->flags); - topic3->sentAssign = true; - } - } - net3::WireEncodeServerHelloDone(out.stream()); - } - Flush(); - m_state = kStateServerHelloComplete; - - // update meta topics - UpdateMetaClientPub(); - UpdateMetaClientSub(); -} - -void ServerImpl::ClientData3::ServerHello(unsigned int flags, - std::string_view self_id) { - DEBUG4("ServerHello({}, {}, {})", m_id, flags, self_id); - m_decoder.SetError("received unexpected ServerHello message"); -} - -void ServerImpl::ClientData3::EntryAssign(std::string_view name, - unsigned int id, unsigned int seq_num, - const Value& value, - unsigned int flags) { - DEBUG4("EntryAssign({}, {}, {}, {}, {})", m_id, id, seq_num, - static_cast(value.type()), flags); - if (id != 0xffff) { - DEBUG3("ignored EntryAssign from {} with non-0xffff id {}", m_id, id); - return; - } - - // convert from NT3 info - auto typeStr = TypeToString(value.type()); - wpi::json properties = wpi::json::object(); - properties["retained"] = true; // treat all NT3 published topics as retained - properties["cached"] = true; // treat all NT3 published topics as cached - if ((flags & NT_PERSISTENT) != 0) { - properties["persistent"] = true; - } - - // create topic - auto topic = m_server.CreateTopic(this, name, typeStr, properties); - TopicData3* topic3 = GetTopic3(topic); - if (topic3->published || topic3->sentAssign) { - WARN("ignoring client {} duplicate publish of '{}'", m_id, name); - return; - } - ++topic3->seqNum; - topic3->published = true; - topic3->pubuid = m_nextPubUid++; - topic3->sentAssign = true; - - // create publisher - auto [publisherIt, isNew] = m_publishers.try_emplace( - topic3->pubuid, - std::make_unique(this, topic, topic3->pubuid)); - if (!isNew) { - return; // shouldn't happen, but just in case... - } - - // add publisher to topic - topic->AddPublisher(this, publisherIt->getSecond().get()); - - // update meta data - m_server.UpdateMetaTopicPub(topic); - UpdateMetaClientPub(); - - // acts as an announce + data update - SendAnnounce(topic, topic3->pubuid); - m_server.SetValue(this, topic, value); - - // respond with assign message with assigned topic ID - if (m_local && m_state == kStateRunning) { - net3::WireEncodeEntryAssign(m_wire.Send().stream(), topic->name, topic->id, - topic3->seqNum.value(), value, topic3->flags); - } else { - m_outgoing.emplace_back(net3::Message3::EntryAssign( - topic->name, topic->id, topic3->seqNum.value(), value, topic3->flags)); - } -} - -void ServerImpl::ClientData3::EntryUpdate(unsigned int id, unsigned int seq_num, - const Value& value) { - DEBUG4("EntryUpdate({}, {}, {}, {})", m_id, id, seq_num, - static_cast(value.type())); - if (m_state != kStateRunning) { - m_decoder.SetError("received unexpected EntryUpdate message"); - return; - } - - if (id >= m_server.m_topics.size()) { - DEBUG3("ignored EntryUpdate from {} on non-existent topic {}", m_id, id); - return; - } - TopicData* topic = m_server.m_topics[id].get(); - if (!topic || !topic->IsPublished()) { - DEBUG3("ignored EntryUpdate from {} on non-existent topic {}", m_id, id); - return; - } - - TopicData3* topic3 = GetTopic3(topic); - if (!topic3->published) { - topic3->published = true; - topic3->pubuid = m_nextPubUid++; - - // create publisher - auto [publisherIt, isNew] = m_publishers.try_emplace( - topic3->pubuid, - std::make_unique(this, topic, topic3->pubuid)); - if (isNew) { - // add publisher to topic - topic->AddPublisher(this, publisherIt->getSecond().get()); - - // update meta data - m_server.UpdateMetaTopicPub(topic); - UpdateMetaClientPub(); - } - } - topic3->seqNum = net3::SequenceNumber{seq_num}; - - m_server.SetValue(this, topic, value); -} - -void ServerImpl::ClientData3::FlagsUpdate(unsigned int id, unsigned int flags) { - DEBUG4("FlagsUpdate({}, {}, {})", m_id, id, flags); - if (m_state != kStateRunning) { - m_decoder.SetError("received unexpected FlagsUpdate message"); - return; - } - if (id >= m_server.m_topics.size()) { - DEBUG3("ignored FlagsUpdate from {} on non-existent topic {}", m_id, id); - return; - } - TopicData* topic = m_server.m_topics[id].get(); - if (!topic || !topic->IsPublished()) { - DEBUG3("ignored FlagsUpdate from {} on non-existent topic {}", m_id, id); - return; - } - if (topic->special) { - DEBUG3("ignored FlagsUpdate from {} on special topic {}", m_id, id); - return; - } - m_server.SetFlags(this, topic, flags); -} - -void ServerImpl::ClientData3::EntryDelete(unsigned int id) { - DEBUG4("EntryDelete({}, {})", m_id, id); - if (m_state != kStateRunning) { - m_decoder.SetError("received unexpected EntryDelete message"); - return; - } - if (id >= m_server.m_topics.size()) { - DEBUG3("ignored EntryDelete from {} on non-existent topic {}", m_id, id); - return; - } - TopicData* topic = m_server.m_topics[id].get(); - if (!topic || !topic->IsPublished()) { - DEBUG3("ignored EntryDelete from {} on non-existent topic {}", m_id, id); - return; - } - if (topic->special) { - DEBUG3("ignored EntryDelete from {} on special topic {}", m_id, id); - return; - } - - auto topic3it = m_topics3.find(topic); - if (topic3it != m_topics3.end()) { - // make sure we send assign the next time - topic3it->second.sentAssign = false; - - // unpublish from this client (if it was previously published) - if (topic3it->second.published) { - topic3it->second.published = false; - auto publisherIt = m_publishers.find(topic3it->second.pubuid); - if (publisherIt != m_publishers.end()) { - // remove publisher from topic - topic->RemovePublisher(this, publisherIt->second.get()); - - // remove publisher from client - m_publishers.erase(publisherIt); - - // update meta data - m_server.UpdateMetaTopicPub(topic); - UpdateMetaClientPub(); - } - } - } - - // set retained=false - m_server.SetProperties(this, topic, {{"retained", false}}); -} - -bool ServerImpl::TopicData::SetProperties(const wpi::json& update) { - if (!update.is_object()) { - return false; - } - bool updated = false; - for (auto&& elem : update.items()) { - if (elem.value().is_null()) { - properties.erase(elem.key()); - } else { - properties[elem.key()] = elem.value(); - } - updated = true; - } - if (updated) { - RefreshProperties(); - } - return updated; -} - -void ServerImpl::TopicData::RefreshProperties() { - persistent = false; - retained = false; - cached = true; - - auto persistentIt = properties.find("persistent"); - if (persistentIt != properties.end()) { - if (auto val = persistentIt->get_ptr()) { - persistent = *val; - } - } - - auto retainedIt = properties.find("retained"); - if (retainedIt != properties.end()) { - if (auto val = retainedIt->get_ptr()) { - retained = *val; - } - } - - auto cachedIt = properties.find("cached"); - if (cachedIt != properties.end()) { - if (auto val = cachedIt->get_ptr()) { - cached = *val; - } - } - - if (!cached) { - lastValue = {}; - lastValueClient = nullptr; - } - - if (!cached && persistent) { - WARN("topic {}: disabling cached property disables persistent storage", - name); - } -} - -bool ServerImpl::TopicData::SetFlags(unsigned int flags_) { - bool updated; - if ((flags_ & NT_PERSISTENT) != 0) { - updated = !persistent; - persistent = true; - properties["persistent"] = true; - } else { - updated = persistent; - persistent = false; - properties.erase("persistent"); - } - if ((flags_ & NT_RETAINED) != 0) { - updated |= !retained; - retained = true; - properties["retained"] = true; - } else { - updated |= retained; - retained = false; - properties.erase("retained"); - } - if ((flags_ & NT_UNCACHED) != 0) { - updated |= cached; - cached = false; - properties["cached"] = false; - lastValue = {}; - lastValueClient = nullptr; - } else { - updated |= !cached; - cached = true; - properties.erase("cached"); - } - if (!cached && persistent) { - WARN("topic {}: disabling cached property disables persistent storage", - name); - } - return updated; -} - -bool ServerImpl::SubscriberData::Matches(std::string_view name, bool special) { - for (auto&& topicName : topicNames) { - if ((!options.prefixMatch && name == topicName) || - (options.prefixMatch && (!special || !topicName.empty()) && - wpi::starts_with(name, topicName))) { - return true; - } - } - return false; -} - ServerImpl::ServerImpl(wpi::Logger& logger) : m_logger{logger} { // local is client 0 - m_clients.emplace_back(std::make_unique(*this, 0, logger)); - m_localClient = static_cast(m_clients.back().get()); + m_clients.emplace_back(std::make_unique(*this, 0, logger)); + m_localClient = static_cast(m_clients.back().get()); } -std::pair ServerImpl::AddClient( - std::string_view name, std::string_view connInfo, bool local, - net::WireConnection& wire, ServerImpl::SetPeriodicFunc setPeriodic) { +std::pair ServerImpl::AddClient(std::string_view name, + std::string_view connInfo, + bool local, + net::WireConnection& wire, + SetPeriodicFunc setPeriodic) { if (name.empty()) { name = "NT4"; } @@ -1232,9 +71,9 @@ std::pair ServerImpl::AddClient( std::string dedupName = fmt::format("{}@{}", name, index); auto& clientData = m_clients[index]; - clientData = std::make_unique(dedupName, connInfo, local, wire, - std::move(setPeriodic), *this, - index, m_logger); + clientData = std::make_unique(dedupName, connInfo, local, wire, + std::move(setPeriodic), *this, + index, m_logger); // create client meta topics clientData->m_metaPub = @@ -1252,8 +91,8 @@ std::pair ServerImpl::AddClient( int ServerImpl::AddClient3(std::string_view connInfo, bool local, net3::WireConnection3& wire, - ServerImpl::Connected3Func connected, - ServerImpl::SetPeriodicFunc setPeriodic) { + Connected3Func connected, + SetPeriodicFunc setPeriodic) { size_t index = m_clients.size(); // find an empty slot; we can't check for duplicates until we get a hello. // just do a linear search as number of clients is typically small (<10) @@ -1267,7 +106,7 @@ int ServerImpl::AddClient3(std::string_view connInfo, bool local, m_clients.emplace_back(); } - m_clients[index] = std::make_unique( + m_clients[index] = std::make_unique( connInfo, local, wire, std::move(connected), std::move(setPeriodic), *this, index, m_logger); @@ -1280,7 +119,7 @@ std::shared_ptr ServerImpl::RemoveClient(int clientId) { auto& client = m_clients[clientId]; // remove all publishers and subscribers for this client - wpi::SmallVector toDelete; + wpi::SmallVector toDelete; for (auto&& topic : m_topics) { bool pubChanged = false; bool subChanged = false; @@ -1693,11 +532,11 @@ std::string ServerImpl::LoadPersistent(std::string_view in) { return allerrors; } -ServerImpl::TopicData* ServerImpl::CreateTopic(ClientData* client, - std::string_view name, - std::string_view typeStr, - const wpi::json& properties, - bool special) { +ServerTopic* ServerImpl::CreateTopic(ServerClient* client, + std::string_view name, + std::string_view typeStr, + const wpi::json& properties, + bool special) { auto& topic = m_nameTopics[name]; if (topic) { if (typeStr != topic->typeStr) { @@ -1709,7 +548,7 @@ ServerImpl::TopicData* ServerImpl::CreateTopic(ClientData* client, } else { // new topic unsigned int id = m_topics.emplace_back( - std::make_unique(m_logger, name, typeStr, properties)); + std::make_unique(m_logger, name, typeStr, properties)); topic = m_topics[id].get(); topic->id = id; topic->special = special; @@ -1720,7 +559,7 @@ ServerImpl::TopicData* ServerImpl::CreateTopic(ClientData* client, } // look for subscriber matching prefixes - wpi::SmallVector subscribersBuf; + wpi::SmallVector subscribersBuf; auto subscribers = aClient->GetSubscribers(name, topic->special, subscribersBuf); @@ -1760,11 +599,11 @@ ServerImpl::TopicData* ServerImpl::CreateTopic(ClientData* client, return topic; } -ServerImpl::TopicData* ServerImpl::CreateMetaTopic(std::string_view name) { +ServerTopic* ServerImpl::CreateMetaTopic(std::string_view name) { return CreateTopic(nullptr, name, "msgpack", {{"retained", true}}, true); } -void ServerImpl::DeleteTopic(TopicData* topic) { +void ServerImpl::DeleteTopic(ServerTopic* topic) { if (!topic) { return; } @@ -1790,7 +629,7 @@ void ServerImpl::DeleteTopic(TopicData* topic) { m_topics.erase(topic->id); } -void ServerImpl::SetProperties(ClientData* client, TopicData* topic, +void ServerImpl::SetProperties(ServerClient* client, ServerTopic* topic, const wpi::json& update) { DEBUG4("SetProperties({}, {}, {})", client ? client->GetId() : -1, topic->name, update.dump()); @@ -1804,7 +643,7 @@ void ServerImpl::SetProperties(ClientData* client, TopicData* topic, } } -void ServerImpl::SetFlags(ClientData* client, TopicData* topic, +void ServerImpl::SetFlags(ServerClient* client, ServerTopic* topic, unsigned int flags) { bool wasPersistent = topic->persistent; if (topic->SetFlags(flags)) { @@ -1822,7 +661,7 @@ void ServerImpl::SetFlags(ClientData* client, TopicData* topic, } } -void ServerImpl::SetValue(ClientData* client, TopicData* topic, +void ServerImpl::SetValue(ServerClient* client, ServerTopic* topic, const Value& value) { // update retained value if from same client or timestamp newer if (topic->cached && (!topic->lastValue || topic->lastValueClient == client || @@ -1868,7 +707,7 @@ void ServerImpl::UpdateMetaClients(const std::vector& conns) { } } -void ServerImpl::UpdateMetaTopicPub(TopicData* topic) { +void ServerImpl::UpdateMetaTopicPub(ServerTopic* topic) { if (!topic->metaPub) { return; } @@ -1891,7 +730,7 @@ void ServerImpl::UpdateMetaTopicPub(TopicData* topic) { } } -void ServerImpl::UpdateMetaTopicSub(TopicData* topic) { +void ServerImpl::UpdateMetaTopicSub(ServerTopic* topic) { if (!topic->metaSub) { return; } @@ -1914,7 +753,7 @@ void ServerImpl::UpdateMetaTopicSub(TopicData* topic) { } } -void ServerImpl::PropertiesChanged(ClientData* client, TopicData* topic, +void ServerImpl::PropertiesChanged(ServerClient* client, ServerTopic* topic, const wpi::json& update) { // removing some properties can result in the topic being unpublished if (!topic->IsPublished()) { diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.h b/ntcore/src/main/native/cpp/server/ServerImpl.h index 55c39826178..1a382dfe577 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.h +++ b/ntcore/src/main/native/cpp/server/ServerImpl.h @@ -6,8 +6,6 @@ #include -#include -#include #include #include #include @@ -15,24 +13,13 @@ #include #include -#include -#include #include #include -#include +#include -#include "PubSubOptions.h" -#include "net/ClientMessageQueue.h" -#include "net/Message.h" -#include "net/NetworkOutgoingQueue.h" -#include "net/NetworkPing.h" -#include "net/WireConnection.h" -#include "net/WireDecoder.h" -#include "net/WireEncoder.h" -#include "net3/Message3.h" -#include "net3/SequenceNumber.h" -#include "net3/WireConnection3.h" -#include "net3/WireDecoder3.h" +#include "server/Functions.h" +#include "server/ServerClient.h" +#include "server/ServerTopic.h" namespace wpi { class Logger; @@ -42,8 +29,9 @@ class raw_ostream; } // namespace wpi namespace nt::net { -struct ClientMessage; +class ClientMessageQueue; class LocalInterface; +class ServerMessageHandler; class WireConnection; } // namespace nt::net @@ -53,12 +41,20 @@ class WireConnection3; namespace nt::server { +class ServerClient; +class ServerClient3; +class ServerClient4; +class ServerClient4Base; +class ServerClientLocal; + class ServerImpl final { - public: - using SetPeriodicFunc = std::function; - using Connected3Func = - std::function; + friend class ServerClient; + friend class ServerClient3; + friend class ServerClient4; + friend class ServerClient4Base; + friend class ServerClientLocal; + public: explicit ServerImpl(wpi::Logger& logger); void SendAllOutgoing(uint64_t curTimeMs, bool flush); @@ -95,423 +91,39 @@ class ServerImpl final { std::string LoadPersistent(std::string_view in); private: - static constexpr uint32_t kMinPeriodMs = 5; - - class ClientData; - struct PublisherData; - struct SubscriberData; - - struct TopicData { - TopicData(wpi::Logger& logger, std::string_view name, - std::string_view typeStr) - : m_logger{logger}, name{name}, typeStr{typeStr} {} - TopicData(wpi::Logger& logger, std::string_view name, - std::string_view typeStr, wpi::json properties) - : m_logger{logger}, - name{name}, - typeStr{typeStr}, - properties(std::move(properties)) { - RefreshProperties(); - } - - bool IsPublished() const { - return persistent || retained || publisherCount != 0; - } - - // returns true if properties changed - bool SetProperties(const wpi::json& update); - void RefreshProperties(); - bool SetFlags(unsigned int flags_); - - wpi::Logger& m_logger; // Must be m_logger for WARN macro to work - std::string name; - unsigned int id; - Value lastValue; - ClientData* lastValueClient = nullptr; - std::string typeStr; - wpi::json properties = wpi::json::object(); - unsigned int publisherCount{0}; - bool persistent{false}; - bool retained{false}; - bool cached{true}; - bool special{false}; - int localTopic{0}; - - void AddPublisher(ClientData* client, PublisherData* pub) { - if (clients[client].publishers.insert(pub).second) { - ++publisherCount; - } - } - - void RemovePublisher(ClientData* client, PublisherData* pub) { - if (clients[client].publishers.erase(pub)) { - --publisherCount; - } - } - - struct TopicClientData { - wpi::SmallPtrSet publishers; - wpi::SmallPtrSet subscribers; - net::ValueSendMode sendMode = net::ValueSendMode::kDisabled; - - bool AddSubscriber(SubscriberData* sub) { - bool added = subscribers.insert(sub).second; - if (!sub->options.topicsOnly) { - if (sub->options.sendAll) { - sendMode = net::ValueSendMode::kAll; - } else if (sendMode == net::ValueSendMode::kDisabled) { - sendMode = net::ValueSendMode::kNormal; - } - } - return added; - } - }; - wpi::SmallDenseMap clients; - - // meta topics - TopicData* metaPub = nullptr; - TopicData* metaSub = nullptr; - }; - - class ClientData { - public: - ClientData(std::string_view name, std::string_view connInfo, bool local, - ServerImpl::SetPeriodicFunc setPeriodic, ServerImpl& server, - int id, wpi::Logger& logger) - : m_name{name}, - m_connInfo{connInfo}, - m_local{local}, - m_setPeriodic{std::move(setPeriodic)}, - m_server{server}, - m_id{id}, - m_logger{logger} {} - virtual ~ClientData() = default; - - // these return true if any messages have been queued for later processing - virtual bool ProcessIncomingText(std::string_view data) = 0; - virtual bool ProcessIncomingBinary(std::span data) = 0; - - virtual void SendValue(TopicData* topic, const Value& value, - net::ValueSendMode mode) = 0; - virtual void SendAnnounce(TopicData* topic, std::optional pubuid) = 0; - virtual void SendUnannounce(TopicData* topic) = 0; - virtual void SendPropertiesUpdate(TopicData* topic, const wpi::json& update, - bool ack) = 0; - virtual void SendOutgoing(uint64_t curTimeMs, bool flush) = 0; - virtual void Flush() = 0; - - // later processing -- returns true if more to process - virtual bool ProcessIncomingMessages(size_t max) = 0; - - void UpdateMetaClientPub(); - void UpdateMetaClientSub(); - - std::span GetSubscribers( - std::string_view name, bool special, - wpi::SmallVectorImpl& buf); - - std::string_view GetName() const { return m_name; } - int GetId() const { return m_id; } - - virtual void UpdatePeriod(TopicData::TopicClientData& tcd, - TopicData* topic) {} - - protected: - std::string m_name; - std::string m_connInfo; - bool m_local; // local to machine - ServerImpl::SetPeriodicFunc m_setPeriodic; - // TODO: make this per-topic? - uint32_t m_periodMs{UINT32_MAX}; - ServerImpl& m_server; - int m_id; - - wpi::Logger& m_logger; - - wpi::DenseMap> m_publishers; - wpi::DenseMap> m_subscribers; - - public: - // meta topics - TopicData* m_metaPub = nullptr; - TopicData* m_metaSub = nullptr; - }; - - class ClientData4Base : public ClientData, - protected net::ClientMessageHandler { - public: - ClientData4Base(std::string_view name, std::string_view connInfo, - bool local, ServerImpl::SetPeriodicFunc setPeriodic, - ServerImpl& server, int id, wpi::Logger& logger) - : ClientData{name, connInfo, local, setPeriodic, server, id, logger} {} - - protected: - // ClientMessageHandler interface - void ClientPublish(int pubuid, std::string_view name, - std::string_view typeStr, const wpi::json& properties, - const PubSubOptionsImpl& options) final; - void ClientUnpublish(int pubuid) final; - void ClientSetProperties(std::string_view name, - const wpi::json& update) final; - void ClientSubscribe(int subuid, std::span topicNames, - const PubSubOptionsImpl& options) final; - void ClientUnsubscribe(int subuid) final; - - void ClientSetValue(int pubuid, const Value& value) final; - - bool DoProcessIncomingMessages(net::ClientMessageQueue& queue, size_t max); - - wpi::DenseMap m_announceSent; - - private: - std::array m_msgsBuf; - }; - - class ClientDataLocal final : public ClientData4Base { - public: - ClientDataLocal(ServerImpl& server, int id, wpi::Logger& logger) - : ClientData4Base{"", "", true, [](uint32_t) {}, server, id, logger} {} - - bool ProcessIncomingText(std::string_view data) final { return false; } - bool ProcessIncomingBinary(std::span data) final { - return false; - } - - bool ProcessIncomingMessages(size_t max) final { - if (!m_queue) { - return false; - } - return DoProcessIncomingMessages(*m_queue, max); - } - - void SendValue(TopicData* topic, const Value& value, - net::ValueSendMode mode) final; - void SendAnnounce(TopicData* topic, std::optional pubuid) final; - void SendUnannounce(TopicData* topic) final; - void SendPropertiesUpdate(TopicData* topic, const wpi::json& update, - bool ack) final; - void SendOutgoing(uint64_t curTimeMs, bool flush) final {} - void Flush() final {} - - void SetQueue(net::ClientMessageQueue* queue) { m_queue = queue; } - - private: - net::ClientMessageQueue* m_queue = nullptr; - }; - - class ClientData4 final : public ClientData4Base { - public: - ClientData4(std::string_view name, std::string_view connInfo, bool local, - net::WireConnection& wire, - ServerImpl::SetPeriodicFunc setPeriodic, ServerImpl& server, - int id, wpi::Logger& logger) - : ClientData4Base{name, connInfo, local, setPeriodic, - server, id, logger}, - m_wire{wire}, - m_ping{wire}, - m_incoming{logger}, - m_outgoing{wire, local} {} - - bool ProcessIncomingText(std::string_view data) final; - bool ProcessIncomingBinary(std::span data) final; - - bool ProcessIncomingMessages(size_t max) final { - if (!DoProcessIncomingMessages(m_incoming, max)) { - m_wire.StartRead(); - return false; - } - return true; - } - - void SendValue(TopicData* topic, const Value& value, - net::ValueSendMode mode) final; - void SendAnnounce(TopicData* topic, std::optional pubuid) final; - void SendUnannounce(TopicData* topic) final; - void SendPropertiesUpdate(TopicData* topic, const wpi::json& update, - bool ack) final; - void SendOutgoing(uint64_t curTimeMs, bool flush) final; - - void Flush() final {} - - void UpdatePeriod(TopicData::TopicClientData& tcd, TopicData* topic) final; - - public: - net::WireConnection& m_wire; - - private: - net::NetworkPing m_ping; - net::NetworkIncomingClientQueue m_incoming; - net::NetworkOutgoingQueue m_outgoing; - }; - - class ClientData3 final : public ClientData, private net3::MessageHandler3 { - public: - ClientData3(std::string_view connInfo, bool local, - net3::WireConnection3& wire, - ServerImpl::Connected3Func connected, - ServerImpl::SetPeriodicFunc setPeriodic, ServerImpl& server, - int id, wpi::Logger& logger) - : ClientData{"", connInfo, local, setPeriodic, server, id, logger}, - m_connected{std::move(connected)}, - m_wire{wire}, - m_decoder{*this}, - m_incoming{logger} {} - - bool ProcessIncomingText(std::string_view data) final { return false; } - bool ProcessIncomingBinary(std::span data) final; - - bool ProcessIncomingMessages(size_t max) final { return false; } - - void SendValue(TopicData* topic, const Value& value, - net::ValueSendMode mode) final; - void SendAnnounce(TopicData* topic, std::optional pubuid) final; - void SendUnannounce(TopicData* topic) final; - void SendPropertiesUpdate(TopicData* topic, const wpi::json& update, - bool ack) final; - void SendOutgoing(uint64_t curTimeMs, bool flush) final; - - void Flush() final { m_wire.Flush(); } - - private: - // MessageHandler3 interface - void KeepAlive() final; - void ServerHelloDone() final; - void ClientHelloDone() final; - void ClearEntries() final; - void ProtoUnsup(unsigned int proto_rev) final; - void ClientHello(std::string_view self_id, unsigned int proto_rev) final; - void ServerHello(unsigned int flags, std::string_view self_id) final; - void EntryAssign(std::string_view name, unsigned int id, - unsigned int seq_num, const Value& value, - unsigned int flags) final; - void EntryUpdate(unsigned int id, unsigned int seq_num, - const Value& value) final; - void FlagsUpdate(unsigned int id, unsigned int flags) final; - void EntryDelete(unsigned int id) final; - void ExecuteRpc(unsigned int id, unsigned int uid, - std::span params) final {} - void RpcResponse(unsigned int id, unsigned int uid, - std::span result) final {} - - ServerImpl::Connected3Func m_connected; - net3::WireConnection3& m_wire; - - enum State { kStateInitial, kStateServerHelloComplete, kStateRunning }; - State m_state{kStateInitial}; - net3::WireDecoder3 m_decoder; - - net::NetworkIncomingClientQueue m_incoming; - std::vector m_outgoing; - wpi::DenseMap m_outgoingValueMap; - int64_t m_nextPubUid{1}; - uint64_t m_lastSendMs{0}; - - struct TopicData3 { - explicit TopicData3(TopicData* topic) { UpdateFlags(topic); } - - unsigned int flags{0}; - net3::SequenceNumber seqNum; - bool sentAssign{false}; - bool published{false}; - int64_t pubuid{0}; - - bool UpdateFlags(TopicData* topic); - }; - wpi::DenseMap m_topics3; - TopicData3* GetTopic3(TopicData* topic) { - return &m_topics3.try_emplace(topic, topic).first->second; - } - }; - - struct PublisherData { - PublisherData(ClientData* client, TopicData* topic, int64_t pubuid) - : client{client}, topic{topic}, pubuid{pubuid} { - UpdateMeta(); - } - - void UpdateMeta(); - - ClientData* client; - TopicData* topic; - int64_t pubuid; - std::vector metaClient; - std::vector metaTopic; - }; - - struct SubscriberData { - SubscriberData(ClientData* client, std::span topicNames, - int64_t subuid, const PubSubOptionsImpl& options) - : client{client}, - topicNames{topicNames.begin(), topicNames.end()}, - subuid{subuid}, - options{options}, - periodMs(std::lround(options.periodicMs / 10.0) * 10) { - UpdateMeta(); - if (periodMs < kMinPeriodMs) { - periodMs = kMinPeriodMs; - } - } - - void Update(std::span topicNames_, - const PubSubOptionsImpl& options_) { - topicNames = {topicNames_.begin(), topicNames_.end()}; - options = options_; - UpdateMeta(); - periodMs = std::lround(options_.periodicMs / 10.0) * 10; - if (periodMs < kMinPeriodMs) { - periodMs = kMinPeriodMs; - } - } - - bool Matches(std::string_view name, bool special); - - void UpdateMeta(); - - ClientData* client; - std::vector topicNames; - int64_t subuid; - PubSubOptionsImpl options; - std::vector metaClient; - std::vector metaTopic; - wpi::DenseMap topics; - // in options as double, but copy here as integer; rounded to the nearest - // 10 ms - uint32_t periodMs; - }; - wpi::Logger& m_logger; net::ServerMessageHandler* m_local{nullptr}; bool m_controlReady{false}; - ClientDataLocal* m_localClient; - std::vector> m_clients; - wpi::UidVector, 16> m_topics; - wpi::StringMap m_nameTopics; + ServerClientLocal* m_localClient; + std::vector> m_clients; + wpi::UidVector, 16> m_topics; + wpi::StringMap m_nameTopics; bool m_persistentChanged{false}; // global meta topics (other meta topics are linked to from the specific // client or topic) - TopicData* m_metaClients; + ServerTopic* m_metaClients; void DumpPersistent(wpi::raw_ostream& os); // helper functions - TopicData* CreateTopic(ClientData* client, std::string_view name, - std::string_view typeStr, const wpi::json& properties, - bool special = false); - TopicData* CreateMetaTopic(std::string_view name); - void DeleteTopic(TopicData* topic); - void SetProperties(ClientData* client, TopicData* topic, + ServerTopic* CreateTopic(ServerClient* client, std::string_view name, + std::string_view typeStr, + const wpi::json& properties, bool special = false); + ServerTopic* CreateMetaTopic(std::string_view name); + void DeleteTopic(ServerTopic* topic); + void SetProperties(ServerClient* client, ServerTopic* topic, const wpi::json& update); - void SetFlags(ClientData* client, TopicData* topic, unsigned int flags); - void SetValue(ClientData* client, TopicData* topic, const Value& value); + void SetFlags(ServerClient* client, ServerTopic* topic, unsigned int flags); + void SetValue(ServerClient* client, ServerTopic* topic, const Value& value); // update meta topic values from data structures void UpdateMetaClients(const std::vector& conns); - void UpdateMetaTopicPub(TopicData* topic); - void UpdateMetaTopicSub(TopicData* topic); + void UpdateMetaTopicPub(ServerTopic* topic); + void UpdateMetaTopicSub(ServerTopic* topic); - void PropertiesChanged(ClientData* client, TopicData* topic, + void PropertiesChanged(ServerClient* client, ServerTopic* topic, const wpi::json& update); }; diff --git a/ntcore/src/main/native/cpp/server/ServerPublisher.cpp b/ntcore/src/main/native/cpp/server/ServerPublisher.cpp new file mode 100644 index 00000000000..04b96f88d91 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerPublisher.cpp @@ -0,0 +1,47 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "ServerPublisher.h" + +#include + +#include + +#include "server/MessagePackWriter.h" +#include "server/ServerClient.h" +#include "server/ServerTopic.h" + +using namespace nt::server; +using namespace mpack; + +void ServerPublisher::UpdateMeta() { + { + Writer w; + mpack_start_map(&w, 2); + mpack_write_str(&w, "uid"); + mpack_write_int(&w, pubuid); + mpack_write_str(&w, "topic"); + mpack_write_str(&w, topic->name); + mpack_finish_map(&w); + if (mpack_writer_destroy(&w) == mpack_ok) { + metaClient = std::move(w.bytes); + } + } + { + Writer w; + mpack_start_map(&w, 2); + mpack_write_str(&w, "client"); + if (client) { + mpack_write_str(&w, client->GetName()); + } else { + mpack_write_str(&w, ""); + } + mpack_write_str(&w, "pubuid"); + mpack_write_int(&w, pubuid); + mpack_finish_map(&w); + if (mpack_writer_destroy(&w) == mpack_ok) { + metaTopic = std::move(w.bytes); + } + } +} diff --git a/ntcore/src/main/native/cpp/server/ServerPublisher.h b/ntcore/src/main/native/cpp/server/ServerPublisher.h new file mode 100644 index 00000000000..ee573c0483d --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerPublisher.h @@ -0,0 +1,31 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include + +namespace nt::server { + +class ServerClient; +struct ServerTopic; + +struct ServerPublisher { + ServerPublisher(ServerClient* client, ServerTopic* topic, int64_t pubuid) + : client{client}, topic{topic}, pubuid{pubuid} { + UpdateMeta(); + } + + void UpdateMeta(); + + ServerClient* client; + ServerTopic* topic; + int64_t pubuid; + std::vector metaClient; + std::vector metaTopic; +}; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp b/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp new file mode 100644 index 00000000000..a593f322e6d --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp @@ -0,0 +1,93 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "ServerSubscriber.h" + +#include + +#include +#include + +#include "PubSubOptions.h" +#include "server/MessagePackWriter.h" +#include "server/ServerClient.h" + +using namespace nt; +using namespace nt::server; +using namespace mpack; + +static void WriteOptions(mpack_writer_t& w, const PubSubOptionsImpl& options) { + int size = + (options.sendAll ? 1 : 0) + (options.topicsOnly ? 1 : 0) + + (options.periodicMs != PubSubOptionsImpl::kDefaultPeriodicMs ? 1 : 0) + + (options.prefixMatch ? 1 : 0); + mpack_start_map(&w, size); + if (options.sendAll) { + mpack_write_str(&w, "all"); + mpack_write_bool(&w, true); + } + if (options.topicsOnly) { + mpack_write_str(&w, "topicsonly"); + mpack_write_bool(&w, true); + } + if (options.periodicMs != PubSubOptionsImpl::kDefaultPeriodicMs) { + mpack_write_str(&w, "periodic"); + mpack_write_float(&w, options.periodicMs / 1000.0); + } + if (options.prefixMatch) { + mpack_write_str(&w, "prefix"); + mpack_write_bool(&w, true); + } + mpack_finish_map(&w); +} + +bool ServerSubscriber::Matches(std::string_view name, bool special) { + for (auto&& topicName : topicNames) { + if ((!options.prefixMatch && name == topicName) || + (options.prefixMatch && (!special || !topicName.empty()) && + wpi::starts_with(name, topicName))) { + return true; + } + } + return false; +} + +void ServerSubscriber::UpdateMeta() { + { + Writer w; + mpack_start_map(&w, 3); + mpack_write_str(&w, "uid"); + mpack_write_int(&w, subuid); + mpack_write_str(&w, "topics"); + mpack_start_array(&w, topicNames.size()); + for (auto&& name : topicNames) { + mpack_write_str(&w, name); + } + mpack_finish_array(&w); + mpack_write_str(&w, "options"); + WriteOptions(w, options); + mpack_finish_map(&w); + if (mpack_writer_destroy(&w) == mpack_ok) { + metaClient = std::move(w.bytes); + } + } + { + Writer w; + mpack_start_map(&w, 3); + mpack_write_str(&w, "client"); + if (client) { + mpack_write_str(&w, client->GetName()); + } else { + mpack_write_str(&w, ""); + } + mpack_write_str(&w, "subuid"); + mpack_write_int(&w, subuid); + mpack_write_str(&w, "options"); + WriteOptions(w, options); + mpack_finish_map(&w); + if (mpack_writer_destroy(&w) == mpack_ok) { + metaTopic = std::move(w.bytes); + } + } +} diff --git a/ntcore/src/main/native/cpp/server/ServerSubscriber.h b/ntcore/src/main/native/cpp/server/ServerSubscriber.h new file mode 100644 index 00000000000..eba76ce20c1 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerSubscriber.h @@ -0,0 +1,66 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include +#include +#include +#include + +#include + +#include "PubSubOptions.h" +#include "server/Constants.h" + +namespace nt::server { + +class ServerClient; +struct ServerTopic; + +struct ServerSubscriber { + ServerSubscriber(ServerClient* client, + std::span topicNames, int64_t subuid, + const PubSubOptionsImpl& options) + : client{client}, + topicNames{topicNames.begin(), topicNames.end()}, + subuid{subuid}, + options{options}, + periodMs(std::lround(options.periodicMs / 10.0) * 10) { + UpdateMeta(); + if (periodMs < kMinPeriodMs) { + periodMs = kMinPeriodMs; + } + } + + void Update(std::span topicNames_, + const PubSubOptionsImpl& options_) { + topicNames = {topicNames_.begin(), topicNames_.end()}; + options = options_; + UpdateMeta(); + periodMs = std::lround(options_.periodicMs / 10.0) * 10; + if (periodMs < kMinPeriodMs) { + periodMs = kMinPeriodMs; + } + } + + bool Matches(std::string_view name, bool special); + + void UpdateMeta(); + + ServerClient* client; + std::vector topicNames; + int64_t subuid; + PubSubOptionsImpl options; + std::vector metaClient; + std::vector metaTopic; + wpi::DenseMap topics; + // in options as double, but copy here as integer; rounded to the nearest + // 10 ms + uint32_t periodMs; +}; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerTopic.cpp b/ntcore/src/main/native/cpp/server/ServerTopic.cpp new file mode 100644 index 00000000000..826a49e9c24 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerTopic.cpp @@ -0,0 +1,103 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "ServerTopic.h" + +#include "Log.h" + +using namespace nt::server; + +bool ServerTopic::SetProperties(const wpi::json& update) { + if (!update.is_object()) { + return false; + } + bool updated = false; + for (auto&& elem : update.items()) { + if (elem.value().is_null()) { + properties.erase(elem.key()); + } else { + properties[elem.key()] = elem.value(); + } + updated = true; + } + if (updated) { + RefreshProperties(); + } + return updated; +} + +void ServerTopic::RefreshProperties() { + persistent = false; + retained = false; + cached = true; + + auto persistentIt = properties.find("persistent"); + if (persistentIt != properties.end()) { + if (auto val = persistentIt->get_ptr()) { + persistent = *val; + } + } + + auto retainedIt = properties.find("retained"); + if (retainedIt != properties.end()) { + if (auto val = retainedIt->get_ptr()) { + retained = *val; + } + } + + auto cachedIt = properties.find("cached"); + if (cachedIt != properties.end()) { + if (auto val = cachedIt->get_ptr()) { + cached = *val; + } + } + + if (!cached) { + lastValue = {}; + lastValueClient = nullptr; + } + + if (!cached && persistent) { + WARN("topic {}: disabling cached property disables persistent storage", + name); + } +} + +bool ServerTopic::SetFlags(unsigned int flags_) { + bool updated; + if ((flags_ & NT_PERSISTENT) != 0) { + updated = !persistent; + persistent = true; + properties["persistent"] = true; + } else { + updated = persistent; + persistent = false; + properties.erase("persistent"); + } + if ((flags_ & NT_RETAINED) != 0) { + updated |= !retained; + retained = true; + properties["retained"] = true; + } else { + updated |= retained; + retained = false; + properties.erase("retained"); + } + if ((flags_ & NT_UNCACHED) != 0) { + updated |= cached; + cached = false; + properties["cached"] = false; + lastValue = {}; + lastValueClient = nullptr; + } else { + updated |= !cached; + cached = true; + properties.erase("cached"); + } + if (!cached && persistent) { + WARN("topic {}: disabling cached property disables persistent storage", + name); + } + return updated; +} diff --git a/ntcore/src/main/native/cpp/server/ServerTopic.h b/ntcore/src/main/native/cpp/server/ServerTopic.h new file mode 100644 index 00000000000..e036a8a6435 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerTopic.h @@ -0,0 +1,102 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include "net/NetworkOutgoingQueue.h" +#include "networktables/NetworkTableValue.h" +#include "server/ServerSubscriber.h" + +namespace wpi { +class Logger; +} // namespace wpi + +namespace nt::server { + +class ServerClient; +struct ServerPublisher; +struct ServerSubscriber; + +struct TopicClientData { + wpi::SmallPtrSet publishers; + wpi::SmallPtrSet subscribers; + net::ValueSendMode sendMode = net::ValueSendMode::kDisabled; + + bool AddSubscriber(ServerSubscriber* sub) { + bool added = subscribers.insert(sub).second; + if (!sub->options.topicsOnly) { + if (sub->options.sendAll) { + sendMode = net::ValueSendMode::kAll; + } else if (sendMode == net::ValueSendMode::kDisabled) { + sendMode = net::ValueSendMode::kNormal; + } + } + return added; + } +}; + +struct ServerTopic { + ServerTopic(wpi::Logger& logger, std::string_view name, + std::string_view typeStr) + : m_logger{logger}, name{name}, typeStr{typeStr} {} + ServerTopic(wpi::Logger& logger, std::string_view name, + std::string_view typeStr, wpi::json properties) + : m_logger{logger}, + name{name}, + typeStr{typeStr}, + properties(std::move(properties)) { + RefreshProperties(); + } + + bool IsPublished() const { + return persistent || retained || publisherCount != 0; + } + + // returns true if properties changed + bool SetProperties(const wpi::json& update); + void RefreshProperties(); + bool SetFlags(unsigned int flags_); + + wpi::Logger& m_logger; // Must be m_logger for WARN macro to work + std::string name; + unsigned int id; + Value lastValue; + ServerClient* lastValueClient = nullptr; + std::string typeStr; + wpi::json properties = wpi::json::object(); + unsigned int publisherCount{0}; + bool persistent{false}; + bool retained{false}; + bool cached{true}; + bool special{false}; + int localTopic{0}; + + void AddPublisher(ServerClient* client, ServerPublisher* pub) { + if (clients[client].publishers.insert(pub).second) { + ++publisherCount; + } + } + + void RemovePublisher(ServerClient* client, ServerPublisher* pub) { + if (clients[client].publishers.erase(pub)) { + --publisherCount; + } + } + + wpi::SmallDenseMap clients; + + // meta topics + ServerTopic* metaPub = nullptr; + ServerTopic* metaSub = nullptr; +}; + +} // namespace nt::server From dad9dd1fcd0a4178c318be262e37b221419c8e1b Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Tue, 15 Oct 2024 23:36:03 -0700 Subject: [PATCH 06/26] [ntcore] Move ServerImpl.m_local to ServerClientLocal --- .../native/cpp/server/ServerClientLocal.cpp | 18 +++++++++--------- .../main/native/cpp/server/ServerClientLocal.h | 7 ++++++- .../src/main/native/cpp/server/ServerImpl.cpp | 3 +-- ntcore/src/main/native/cpp/server/ServerImpl.h | 2 -- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp b/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp index 557a30f11b3..36c99d9ae1d 100644 --- a/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp @@ -10,43 +10,43 @@ using namespace nt::server; void ServerClientLocal::SendValue(ServerTopic* topic, const Value& value, net::ValueSendMode mode) { - if (m_server.m_local) { - m_server.m_local->ServerSetValue(topic->localTopic, value); + if (m_local) { + m_local->ServerSetValue(topic->localTopic, value); } } void ServerClientLocal::SendAnnounce(ServerTopic* topic, std::optional pubuid) { - if (m_server.m_local) { + if (m_local) { auto& sent = m_announceSent[topic]; if (sent) { return; } sent = true; - topic->localTopic = m_server.m_local->ServerAnnounce( - topic->name, 0, topic->typeStr, topic->properties, pubuid); + topic->localTopic = m_local->ServerAnnounce(topic->name, 0, topic->typeStr, + topic->properties, pubuid); } } void ServerClientLocal::SendUnannounce(ServerTopic* topic) { - if (m_server.m_local) { + if (m_local) { auto& sent = m_announceSent[topic]; if (!sent) { return; } sent = false; - m_server.m_local->ServerUnannounce(topic->name, topic->localTopic); + m_local->ServerUnannounce(topic->name, topic->localTopic); } } void ServerClientLocal::SendPropertiesUpdate(ServerTopic* topic, const wpi::json& update, bool ack) { - if (m_server.m_local) { + if (m_local) { if (!m_announceSent.lookup(topic)) { return; } - m_server.m_local->ServerPropertiesUpdate(topic->name, update, ack); + m_local->ServerPropertiesUpdate(topic->name, update, ack); } } diff --git a/ntcore/src/main/native/cpp/server/ServerClientLocal.h b/ntcore/src/main/native/cpp/server/ServerClientLocal.h index 60895cf1b5d..0d297cd0b4a 100644 --- a/ntcore/src/main/native/cpp/server/ServerClientLocal.h +++ b/ntcore/src/main/native/cpp/server/ServerClientLocal.h @@ -37,9 +37,14 @@ class ServerClientLocal final : public ServerClient4Base { void SendOutgoing(uint64_t curTimeMs, bool flush) final {} void Flush() final {} - void SetQueue(net::ClientMessageQueue* queue) { m_queue = queue; } + void SetLocal(net::ServerMessageHandler* local, + net::ClientMessageQueue* queue) { + m_local = local; + m_queue = queue; + } private: + net::ServerMessageHandler* m_local = nullptr; net::ClientMessageQueue* m_queue = nullptr; }; diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.cpp b/ntcore/src/main/native/cpp/server/ServerImpl.cpp index 63f48d5752b..75309c3adf4 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.cpp +++ b/ntcore/src/main/native/cpp/server/ServerImpl.cpp @@ -783,8 +783,7 @@ void ServerImpl::SendOutgoing(int clientId, uint64_t curTimeMs) { void ServerImpl::SetLocal(net::ServerMessageHandler* local, net::ClientMessageQueue* queue) { DEBUG4("SetLocal()"); - m_local = local; - m_localClient->SetQueue(queue); + m_localClient->SetLocal(local, queue); // create server meta topics m_metaClients = CreateMetaTopic("$clients"); diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.h b/ntcore/src/main/native/cpp/server/ServerImpl.h index 1a382dfe577..d3e0afa6a6f 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.h +++ b/ntcore/src/main/native/cpp/server/ServerImpl.h @@ -52,7 +52,6 @@ class ServerImpl final { friend class ServerClient3; friend class ServerClient4; friend class ServerClient4Base; - friend class ServerClientLocal; public: explicit ServerImpl(wpi::Logger& logger); @@ -92,7 +91,6 @@ class ServerImpl final { private: wpi::Logger& m_logger; - net::ServerMessageHandler* m_local{nullptr}; bool m_controlReady{false}; ServerClientLocal* m_localClient; From 70a336a84b7a9150edd21efc0b752c23f3c84d02 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Wed, 16 Oct 2024 16:14:51 -0700 Subject: [PATCH 07/26] [ntcore] Store client name instead of pointer in server pub/sub This breaks the circular dependency between publisher/subscriber and client. --- ntcore/src/main/native/cpp/server/ServerClient3.cpp | 6 +++--- ntcore/src/main/native/cpp/server/ServerClient4Base.cpp | 5 +++-- ntcore/src/main/native/cpp/server/ServerPublisher.cpp | 7 +------ ntcore/src/main/native/cpp/server/ServerPublisher.h | 9 ++++++--- ntcore/src/main/native/cpp/server/ServerSubscriber.cpp | 7 +------ ntcore/src/main/native/cpp/server/ServerSubscriber.h | 7 ++++--- 6 files changed, 18 insertions(+), 23 deletions(-) diff --git a/ntcore/src/main/native/cpp/server/ServerClient3.cpp b/ntcore/src/main/native/cpp/server/ServerClient3.cpp index 600e58f17bb..52b175c7f7b 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient3.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient3.cpp @@ -284,7 +284,7 @@ void ServerClient3::ClientHello(std::string_view self_id, PubSubOptions options; options.prefixMatch = true; sub = std::make_unique( - this, std::span{{prefix}}, 0, options); + GetName(), std::span{{prefix}}, 0, options); m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->periodMs); m_setPeriodic(m_periodMs); @@ -356,7 +356,7 @@ void ServerClient3::EntryAssign(std::string_view name, unsigned int id, // create publisher auto [publisherIt, isNew] = m_publishers.try_emplace( topic3->pubuid, - std::make_unique(this, topic, topic3->pubuid)); + std::make_unique(GetName(), topic, topic3->pubuid)); if (!isNew) { return; // shouldn't happen, but just in case... } @@ -409,7 +409,7 @@ void ServerClient3::EntryUpdate(unsigned int id, unsigned int seq_num, // create publisher auto [publisherIt, isNew] = m_publishers.try_emplace( topic3->pubuid, - std::make_unique(this, topic, topic3->pubuid)); + std::make_unique(GetName(), topic, topic3->pubuid)); if (isNew) { // add publisher to topic topic->AddPublisher(this, publisherIt->getSecond().get()); diff --git a/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp b/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp index 5f8a894526e..628ad6afc7f 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp @@ -27,7 +27,7 @@ void ServerClient4Base::ClientPublish(int pubuid, std::string_view name, // create publisher auto [publisherIt, isNew] = m_publishers.try_emplace( - pubuid, std::make_unique(this, topic, pubuid)); + pubuid, std::make_unique(GetName(), topic, pubuid)); if (!isNew) { WARN("client {} duplicate publish of pubuid {}", m_id, pubuid); } else { @@ -101,7 +101,8 @@ void ServerClient4Base::ClientSubscribe(int subuid, replace = true; } else { // create - sub = std::make_unique(this, topicNames, subuid, options); + sub = std::make_unique(GetName(), topicNames, subuid, + options); } // limit subscriber min period diff --git a/ntcore/src/main/native/cpp/server/ServerPublisher.cpp b/ntcore/src/main/native/cpp/server/ServerPublisher.cpp index 04b96f88d91..275acb6b597 100644 --- a/ntcore/src/main/native/cpp/server/ServerPublisher.cpp +++ b/ntcore/src/main/native/cpp/server/ServerPublisher.cpp @@ -9,7 +9,6 @@ #include #include "server/MessagePackWriter.h" -#include "server/ServerClient.h" #include "server/ServerTopic.h" using namespace nt::server; @@ -32,11 +31,7 @@ void ServerPublisher::UpdateMeta() { Writer w; mpack_start_map(&w, 2); mpack_write_str(&w, "client"); - if (client) { - mpack_write_str(&w, client->GetName()); - } else { - mpack_write_str(&w, ""); - } + mpack_write_str(&w, clientName); mpack_write_str(&w, "pubuid"); mpack_write_int(&w, pubuid); mpack_finish_map(&w); diff --git a/ntcore/src/main/native/cpp/server/ServerPublisher.h b/ntcore/src/main/native/cpp/server/ServerPublisher.h index ee573c0483d..b2cfff3077f 100644 --- a/ntcore/src/main/native/cpp/server/ServerPublisher.h +++ b/ntcore/src/main/native/cpp/server/ServerPublisher.h @@ -6,6 +6,8 @@ #include +#include +#include #include namespace nt::server { @@ -14,14 +16,15 @@ class ServerClient; struct ServerTopic; struct ServerPublisher { - ServerPublisher(ServerClient* client, ServerTopic* topic, int64_t pubuid) - : client{client}, topic{topic}, pubuid{pubuid} { + ServerPublisher(std::string_view clientName, ServerTopic* topic, + int64_t pubuid) + : clientName{clientName}, topic{topic}, pubuid{pubuid} { UpdateMeta(); } void UpdateMeta(); - ServerClient* client; + std::string clientName; ServerTopic* topic; int64_t pubuid; std::vector metaClient; diff --git a/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp b/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp index a593f322e6d..3ad8d72c2e8 100644 --- a/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp +++ b/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp @@ -11,7 +11,6 @@ #include "PubSubOptions.h" #include "server/MessagePackWriter.h" -#include "server/ServerClient.h" using namespace nt; using namespace nt::server; @@ -76,11 +75,7 @@ void ServerSubscriber::UpdateMeta() { Writer w; mpack_start_map(&w, 3); mpack_write_str(&w, "client"); - if (client) { - mpack_write_str(&w, client->GetName()); - } else { - mpack_write_str(&w, ""); - } + mpack_write_str(&w, clientName); mpack_write_str(&w, "subuid"); mpack_write_int(&w, subuid); mpack_write_str(&w, "options"); diff --git a/ntcore/src/main/native/cpp/server/ServerSubscriber.h b/ntcore/src/main/native/cpp/server/ServerSubscriber.h index eba76ce20c1..80deda95f27 100644 --- a/ntcore/src/main/native/cpp/server/ServerSubscriber.h +++ b/ntcore/src/main/native/cpp/server/ServerSubscriber.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -22,10 +23,10 @@ class ServerClient; struct ServerTopic; struct ServerSubscriber { - ServerSubscriber(ServerClient* client, + ServerSubscriber(std::string_view clientName, std::span topicNames, int64_t subuid, const PubSubOptionsImpl& options) - : client{client}, + : clientName{clientName}, topicNames{topicNames.begin(), topicNames.end()}, subuid{subuid}, options{options}, @@ -51,7 +52,7 @@ struct ServerSubscriber { void UpdateMeta(); - ServerClient* client; + std::string clientName; std::vector topicNames; int64_t subuid; PubSubOptionsImpl options; From 20654164866e270745f78d2fb69908c2380e3d54 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Wed, 16 Oct 2024 16:36:19 -0700 Subject: [PATCH 08/26] [ntcore] Make server publisher and subscriber classes Add accessor functions where necessary. --- .../main/native/cpp/server/ServerClient.cpp | 8 +-- .../main/native/cpp/server/ServerClient3.cpp | 2 +- .../main/native/cpp/server/ServerClient4.cpp | 4 +- .../native/cpp/server/ServerClient4Base.cpp | 16 ++---- .../src/main/native/cpp/server/ServerImpl.cpp | 8 +-- .../native/cpp/server/ServerPublisher.cpp | 12 ++--- .../main/native/cpp/server/ServerPublisher.h | 21 +++++--- .../native/cpp/server/ServerSubscriber.cpp | 24 ++++----- .../main/native/cpp/server/ServerSubscriber.h | 51 ++++++++++--------- .../src/main/native/cpp/server/ServerTopic.h | 9 ++-- .../src/main/native/include/wpi/MessagePack.h | 6 +++ 11 files changed, 83 insertions(+), 78 deletions(-) diff --git a/ntcore/src/main/native/cpp/server/ServerClient.cpp b/ntcore/src/main/native/cpp/server/ServerClient.cpp index 5f18149384e..92085f2e44e 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient.cpp @@ -22,9 +22,7 @@ void ServerClient::UpdateMetaClientPub() { Writer w; mpack_start_array(&w, m_publishers.size()); for (auto&& pub : m_publishers) { - mpack_write_object_bytes( - &w, reinterpret_cast(pub.second->metaClient.data()), - pub.second->metaClient.size()); + mpack_write_object_bytes(&w, pub.second->GetMetaClientData()); } mpack_finish_array(&w); if (mpack_writer_destroy(&w) == mpack_ok) { @@ -39,9 +37,7 @@ void ServerClient::UpdateMetaClientSub() { Writer w; mpack_start_array(&w, m_subscribers.size()); for (auto&& sub : m_subscribers) { - mpack_write_object_bytes( - &w, reinterpret_cast(sub.second->metaClient.data()), - sub.second->metaClient.size()); + mpack_write_object_bytes(&w, sub.second->GetMetaClientData()); } mpack_finish_array(&w); if (mpack_writer_destroy(&w) == mpack_ok) { diff --git a/ntcore/src/main/native/cpp/server/ServerClient3.cpp b/ntcore/src/main/native/cpp/server/ServerClient3.cpp index 52b175c7f7b..ff47581834d 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient3.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient3.cpp @@ -285,7 +285,7 @@ void ServerClient3::ClientHello(std::string_view self_id, options.prefixMatch = true; sub = std::make_unique( GetName(), std::span{{prefix}}, 0, options); - m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->periodMs); + m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->GetPeriodMs()); m_setPeriodic(m_periodMs); { diff --git a/ntcore/src/main/native/cpp/server/ServerClient4.cpp b/ntcore/src/main/native/cpp/server/ServerClient4.cpp index 5dd8c21c5e1..abcf407a5da 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient4.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient4.cpp @@ -156,8 +156,8 @@ void ServerClient4::SendOutgoing(uint64_t curTimeMs, bool flush) { } void ServerClient4::UpdatePeriod(TopicClientData& tcd, ServerTopic* topic) { - uint32_t period = net::CalculatePeriod(tcd.subscribers, - [](auto& x) { return x->periodMs; }); + uint32_t period = net::CalculatePeriod( + tcd.subscribers, [](auto& x) { return x->GetPeriodMs(); }); DEBUG4("updating {} period to {} ms", topic->name, period); m_outgoing.SetPeriod(topic->id, period); } diff --git a/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp b/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp index 628ad6afc7f..601fdf94c37 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp @@ -12,7 +12,6 @@ #include #include "Log.h" -#include "server/Constants.h" #include "server/ServerImpl.h" #include "server/ServerPublisher.h" @@ -50,7 +49,7 @@ void ServerClient4Base::ClientUnpublish(int pubuid) { return; // nothing to do } auto publisher = publisherIt->getSecond().get(); - auto topic = publisher->topic; + auto topic = publisher->GetTopic(); // remove publisher from topic topic->RemovePublisher(this, publisher); @@ -105,14 +104,9 @@ void ServerClient4Base::ClientSubscribe(int subuid, options); } - // limit subscriber min period - if (sub->periodMs < kMinPeriodMs) { - sub->periodMs = kMinPeriodMs; - } - // update periodic sender (if not local) if (!m_local) { - m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->periodMs); + m_periodMs = net::UpdatePeriodCalc(m_periodMs, sub->GetPeriodMs()); m_setPeriodic(m_periodMs); } @@ -155,7 +149,7 @@ void ServerClient4Base::ClientSubscribe(int subuid, } // send last value - if (added && !sub->options.topicsOnly && !wasSubscribedValue && + if (added && !sub->GetOptions().topicsOnly && !wasSubscribedValue && topic->lastValue) { dataToSend.emplace_back(topic.get()); } @@ -192,7 +186,7 @@ void ServerClient4Base::ClientUnsubscribe(int subuid) { // loop over all subscribers to update period if (!m_local) { m_periodMs = net::CalculatePeriod( - m_subscribers, [](auto& x) { return x.getSecond()->periodMs; }); + m_subscribers, [](auto& x) { return x.getSecond()->GetPeriodMs(); }); m_setPeriodic(m_periodMs); } } @@ -204,7 +198,7 @@ void ServerClient4Base::ClientSetValue(int pubuid, const Value& value) { WARN("unrecognized client {} pubuid {}, ignoring set", m_id, pubuid); return; // ignore unrecognized pubuids } - auto topic = publisherIt->getSecond().get()->topic; + auto topic = publisherIt->getSecond().get()->GetTopic(); m_server.SetValue(this, topic, value); } diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.cpp b/ntcore/src/main/native/cpp/server/ServerImpl.cpp index 75309c3adf4..c34d3d96512 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.cpp +++ b/ntcore/src/main/native/cpp/server/ServerImpl.cpp @@ -719,9 +719,7 @@ void ServerImpl::UpdateMetaTopicPub(ServerTopic* topic) { mpack_start_array(&w, count); for (auto&& tcd : topic->clients) { for (auto&& pub : tcd.second.publishers) { - mpack_write_object_bytes( - &w, reinterpret_cast(pub->metaTopic.data()), - pub->metaTopic.size()); + mpack_write_object_bytes(&w, pub->GetMetaTopicData()); } } mpack_finish_array(&w); @@ -742,9 +740,7 @@ void ServerImpl::UpdateMetaTopicSub(ServerTopic* topic) { mpack_start_array(&w, count); for (auto&& tcd : topic->clients) { for (auto&& sub : tcd.second.subscribers) { - mpack_write_object_bytes( - &w, reinterpret_cast(sub->metaTopic.data()), - sub->metaTopic.size()); + mpack_write_object_bytes(&w, sub->GetMetaTopicData()); } } mpack_finish_array(&w); diff --git a/ntcore/src/main/native/cpp/server/ServerPublisher.cpp b/ntcore/src/main/native/cpp/server/ServerPublisher.cpp index 275acb6b597..90d7b2ea501 100644 --- a/ntcore/src/main/native/cpp/server/ServerPublisher.cpp +++ b/ntcore/src/main/native/cpp/server/ServerPublisher.cpp @@ -19,24 +19,24 @@ void ServerPublisher::UpdateMeta() { Writer w; mpack_start_map(&w, 2); mpack_write_str(&w, "uid"); - mpack_write_int(&w, pubuid); + mpack_write_int(&w, m_pubuid); mpack_write_str(&w, "topic"); - mpack_write_str(&w, topic->name); + mpack_write_str(&w, m_topic->name); mpack_finish_map(&w); if (mpack_writer_destroy(&w) == mpack_ok) { - metaClient = std::move(w.bytes); + m_metaClient = std::move(w.bytes); } } { Writer w; mpack_start_map(&w, 2); mpack_write_str(&w, "client"); - mpack_write_str(&w, clientName); + mpack_write_str(&w, m_clientName); mpack_write_str(&w, "pubuid"); - mpack_write_int(&w, pubuid); + mpack_write_int(&w, m_pubuid); mpack_finish_map(&w); if (mpack_writer_destroy(&w) == mpack_ok) { - metaTopic = std::move(w.bytes); + m_metaTopic = std::move(w.bytes); } } } diff --git a/ntcore/src/main/native/cpp/server/ServerPublisher.h b/ntcore/src/main/native/cpp/server/ServerPublisher.h index b2cfff3077f..dcfe3dc74c1 100644 --- a/ntcore/src/main/native/cpp/server/ServerPublisher.h +++ b/ntcore/src/main/native/cpp/server/ServerPublisher.h @@ -6,6 +6,7 @@ #include +#include #include #include #include @@ -15,20 +16,26 @@ namespace nt::server { class ServerClient; struct ServerTopic; -struct ServerPublisher { +class ServerPublisher { + public: ServerPublisher(std::string_view clientName, ServerTopic* topic, int64_t pubuid) - : clientName{clientName}, topic{topic}, pubuid{pubuid} { + : m_clientName{clientName}, m_topic{topic}, m_pubuid{pubuid} { UpdateMeta(); } + ServerTopic* GetTopic() const { return m_topic; } + std::span GetMetaClientData() const { return m_metaClient; } + std::span GetMetaTopicData() const { return m_metaTopic; } + + private: void UpdateMeta(); - std::string clientName; - ServerTopic* topic; - int64_t pubuid; - std::vector metaClient; - std::vector metaTopic; + std::string m_clientName; + ServerTopic* m_topic; + int64_t m_pubuid; + std::vector m_metaClient; + std::vector m_metaTopic; }; } // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp b/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp index 3ad8d72c2e8..a5acc4f065d 100644 --- a/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp +++ b/ntcore/src/main/native/cpp/server/ServerSubscriber.cpp @@ -42,9 +42,9 @@ static void WriteOptions(mpack_writer_t& w, const PubSubOptionsImpl& options) { } bool ServerSubscriber::Matches(std::string_view name, bool special) { - for (auto&& topicName : topicNames) { - if ((!options.prefixMatch && name == topicName) || - (options.prefixMatch && (!special || !topicName.empty()) && + for (auto&& topicName : m_topicNames) { + if ((!m_options.prefixMatch && name == topicName) || + (m_options.prefixMatch && (!special || !topicName.empty()) && wpi::starts_with(name, topicName))) { return true; } @@ -57,32 +57,32 @@ void ServerSubscriber::UpdateMeta() { Writer w; mpack_start_map(&w, 3); mpack_write_str(&w, "uid"); - mpack_write_int(&w, subuid); + mpack_write_int(&w, m_subuid); mpack_write_str(&w, "topics"); - mpack_start_array(&w, topicNames.size()); - for (auto&& name : topicNames) { + mpack_start_array(&w, m_topicNames.size()); + for (auto&& name : m_topicNames) { mpack_write_str(&w, name); } mpack_finish_array(&w); mpack_write_str(&w, "options"); - WriteOptions(w, options); + WriteOptions(w, m_options); mpack_finish_map(&w); if (mpack_writer_destroy(&w) == mpack_ok) { - metaClient = std::move(w.bytes); + m_metaClient = std::move(w.bytes); } } { Writer w; mpack_start_map(&w, 3); mpack_write_str(&w, "client"); - mpack_write_str(&w, clientName); + mpack_write_str(&w, m_clientName); mpack_write_str(&w, "subuid"); - mpack_write_int(&w, subuid); + mpack_write_int(&w, m_subuid); mpack_write_str(&w, "options"); - WriteOptions(w, options); + WriteOptions(w, m_options); mpack_finish_map(&w); if (mpack_writer_destroy(&w) == mpack_ok) { - metaTopic = std::move(w.bytes); + m_metaTopic = std::move(w.bytes); } } } diff --git a/ntcore/src/main/native/cpp/server/ServerSubscriber.h b/ntcore/src/main/native/cpp/server/ServerSubscriber.h index 80deda95f27..d59b7590193 100644 --- a/ntcore/src/main/native/cpp/server/ServerSubscriber.h +++ b/ntcore/src/main/native/cpp/server/ServerSubscriber.h @@ -12,8 +12,6 @@ #include #include -#include - #include "PubSubOptions.h" #include "server/Constants.h" @@ -22,46 +20,53 @@ namespace nt::server { class ServerClient; struct ServerTopic; -struct ServerSubscriber { +class ServerSubscriber { + public: ServerSubscriber(std::string_view clientName, std::span topicNames, int64_t subuid, const PubSubOptionsImpl& options) - : clientName{clientName}, - topicNames{topicNames.begin(), topicNames.end()}, - subuid{subuid}, - options{options}, - periodMs(std::lround(options.periodicMs / 10.0) * 10) { + : m_clientName{clientName}, + m_topicNames{topicNames.begin(), topicNames.end()}, + m_subuid{subuid}, + m_options{options}, + m_periodMs(std::lround(options.periodicMs / 10.0) * 10) { UpdateMeta(); - if (periodMs < kMinPeriodMs) { - periodMs = kMinPeriodMs; + if (m_periodMs < kMinPeriodMs) { + m_periodMs = kMinPeriodMs; } } void Update(std::span topicNames_, const PubSubOptionsImpl& options_) { - topicNames = {topicNames_.begin(), topicNames_.end()}; - options = options_; + m_topicNames = {topicNames_.begin(), topicNames_.end()}; + m_options = options_; UpdateMeta(); - periodMs = std::lround(options_.periodicMs / 10.0) * 10; - if (periodMs < kMinPeriodMs) { - periodMs = kMinPeriodMs; + m_periodMs = std::lround(options_.periodicMs / 10.0) * 10; + if (m_periodMs < kMinPeriodMs) { + m_periodMs = kMinPeriodMs; } } bool Matches(std::string_view name, bool special); + const PubSubOptions& GetOptions() const { return m_options; } + uint32_t GetPeriodMs() const { return m_periodMs; } + + std::span GetMetaClientData() const { return m_metaClient; } + std::span GetMetaTopicData() const { return m_metaTopic; } + + private: void UpdateMeta(); - std::string clientName; - std::vector topicNames; - int64_t subuid; - PubSubOptionsImpl options; - std::vector metaClient; - std::vector metaTopic; - wpi::DenseMap topics; + std::string m_clientName; + std::vector m_topicNames; + int64_t m_subuid; + PubSubOptionsImpl m_options; + std::vector m_metaClient; + std::vector m_metaTopic; // in options as double, but copy here as integer; rounded to the nearest // 10 ms - uint32_t periodMs; + uint32_t m_periodMs; }; } // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerTopic.h b/ntcore/src/main/native/cpp/server/ServerTopic.h index e036a8a6435..d1ca5c77dc6 100644 --- a/ntcore/src/main/native/cpp/server/ServerTopic.h +++ b/ntcore/src/main/native/cpp/server/ServerTopic.h @@ -23,8 +23,8 @@ class Logger; namespace nt::server { class ServerClient; -struct ServerPublisher; -struct ServerSubscriber; +class ServerPublisher; +class ServerSubscriber; struct TopicClientData { wpi::SmallPtrSet publishers; @@ -33,8 +33,9 @@ struct TopicClientData { bool AddSubscriber(ServerSubscriber* sub) { bool added = subscribers.insert(sub).second; - if (!sub->options.topicsOnly) { - if (sub->options.sendAll) { + auto& options = sub->GetOptions(); + if (!options.topicsOnly) { + if (options.sendAll) { sendMode = net::ValueSendMode::kAll; } else if (sendMode == net::ValueSendMode::kDisabled) { sendMode = net::ValueSendMode::kNormal; diff --git a/wpiutil/src/main/native/include/wpi/MessagePack.h b/wpiutil/src/main/native/include/wpi/MessagePack.h index e1f54e52578..2250c475273 100644 --- a/wpiutil/src/main/native/include/wpi/MessagePack.h +++ b/wpiutil/src/main/native/include/wpi/MessagePack.h @@ -24,6 +24,12 @@ inline void mpack_write_bytes(mpack_writer_t* writer, data.size()); } +inline void mpack_write_object_bytes(mpack_writer_t* writer, + std::span data) { + mpack_write_object_bytes(writer, reinterpret_cast(data.data()), + data.size()); +} + inline void mpack_reader_init_data(mpack_reader_t* reader, std::span data) { mpack_reader_init_data(reader, reinterpret_cast(data.data()), From 39d75b96da72dbdc4d0a14c112e56f03ddc71a40 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Wed, 16 Oct 2024 16:47:33 -0700 Subject: [PATCH 09/26] [ntcore] Server: remove unused m_controlReady --- ntcore/src/main/native/cpp/server/ServerClient4.cpp | 4 ---- ntcore/src/main/native/cpp/server/ServerImpl.h | 1 - 2 files changed, 5 deletions(-) diff --git a/ntcore/src/main/native/cpp/server/ServerClient4.cpp b/ntcore/src/main/native/cpp/server/ServerClient4.cpp index abcf407a5da..f96844f8c6f 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient4.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient4.cpp @@ -10,7 +10,6 @@ #include "Log.h" #include "net/WireDecoder.h" -#include "server/ServerImpl.h" #include "server/ServerTopic.h" using namespace nt::server; @@ -97,7 +96,6 @@ void ServerClient4::SendAnnounce(ServerTopic* topic, m_outgoing.SendMessage( topic->id, net::AnnounceMsg{topic->name, static_cast(topic->id), topic->typeStr, pubuid, topic->properties}); - m_server.m_controlReady = true; } void ServerClient4::SendUnannounce(ServerTopic* topic) { @@ -121,7 +119,6 @@ void ServerClient4::SendUnannounce(ServerTopic* topic) { m_outgoing.SendMessage( topic->id, net::UnannounceMsg{topic->name, static_cast(topic->id)}); m_outgoing.EraseId(topic->id); - m_server.m_controlReady = true; } void ServerClient4::SendPropertiesUpdate(ServerTopic* topic, @@ -143,7 +140,6 @@ void ServerClient4::SendPropertiesUpdate(ServerTopic* topic, } m_outgoing.SendMessage(topic->id, net::PropertiesUpdateMsg{topic->name, update, ack}); - m_server.m_controlReady = true; } void ServerClient4::SendOutgoing(uint64_t curTimeMs, bool flush) { diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.h b/ntcore/src/main/native/cpp/server/ServerImpl.h index d3e0afa6a6f..65a97d50333 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.h +++ b/ntcore/src/main/native/cpp/server/ServerImpl.h @@ -91,7 +91,6 @@ class ServerImpl final { private: wpi::Logger& m_logger; - bool m_controlReady{false}; ServerClientLocal* m_localClient; std::vector> m_clients; From c08be9eab3ca34252209e2bb54366ab88c00667a Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Wed, 16 Oct 2024 23:04:27 -0700 Subject: [PATCH 10/26] [ntcore] Split ServerStorage out from ServerImpl --- .../main/native/cpp/server/ServerClient.cpp | 6 +- .../src/main/native/cpp/server/ServerClient.h | 10 +- .../main/native/cpp/server/ServerClient3.cpp | 52 +- .../main/native/cpp/server/ServerClient3.h | 4 +- .../main/native/cpp/server/ServerClient4.h | 6 +- .../native/cpp/server/ServerClient4Base.cpp | 40 +- .../native/cpp/server/ServerClient4Base.h | 6 +- .../native/cpp/server/ServerClientLocal.h | 4 +- .../src/main/native/cpp/server/ServerImpl.cpp | 672 ++---------------- .../src/main/native/cpp/server/ServerImpl.h | 48 +- .../main/native/cpp/server/ServerPublisher.h | 2 + .../main/native/cpp/server/ServerStorage.cpp | 603 ++++++++++++++++ .../main/native/cpp/server/ServerStorage.h | 90 +++ .../main/native/cpp/server/ServerSubscriber.h | 2 + .../src/main/native/cpp/server/ServerTopic.h | 2 + 15 files changed, 807 insertions(+), 740 deletions(-) create mode 100644 ntcore/src/main/native/cpp/server/ServerStorage.cpp create mode 100644 ntcore/src/main/native/cpp/server/ServerStorage.h diff --git a/ntcore/src/main/native/cpp/server/ServerClient.cpp b/ntcore/src/main/native/cpp/server/ServerClient.cpp index 92085f2e44e..d192668fe9d 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient.cpp @@ -9,8 +9,8 @@ #include #include "server/MessagePackWriter.h" -#include "server/ServerImpl.h" #include "server/ServerPublisher.h" +#include "server/ServerStorage.h" using namespace nt::server; using namespace mpack; @@ -26,7 +26,7 @@ void ServerClient::UpdateMetaClientPub() { } mpack_finish_array(&w); if (mpack_writer_destroy(&w) == mpack_ok) { - m_server.SetValue(nullptr, m_metaPub, Value::MakeRaw(std::move(w.bytes))); + m_storage.SetValue(nullptr, m_metaPub, Value::MakeRaw(std::move(w.bytes))); } } @@ -41,7 +41,7 @@ void ServerClient::UpdateMetaClientSub() { } mpack_finish_array(&w); if (mpack_writer_destroy(&w) == mpack_ok) { - m_server.SetValue(nullptr, m_metaSub, Value::MakeRaw(std::move(w.bytes))); + m_storage.SetValue(nullptr, m_metaSub, Value::MakeRaw(std::move(w.bytes))); } } diff --git a/ntcore/src/main/native/cpp/server/ServerClient.h b/ntcore/src/main/native/cpp/server/ServerClient.h index 2c2f496a860..d56753b75b9 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient.h +++ b/ntcore/src/main/native/cpp/server/ServerClient.h @@ -29,21 +29,23 @@ class SmallVectorImpl; namespace nt::server { struct ServerTopic; -class ServerImpl; +class ServerStorage; struct TopicClientData; class ServerClient { public: ServerClient(std::string_view name, std::string_view connInfo, bool local, - SetPeriodicFunc setPeriodic, ServerImpl& server, int id, + SetPeriodicFunc setPeriodic, ServerStorage& storage, int id, wpi::Logger& logger) : m_name{name}, m_connInfo{connInfo}, m_local{local}, m_setPeriodic{std::move(setPeriodic)}, - m_server{server}, + m_storage{storage}, m_id{id}, m_logger{logger} {} + ServerClient(const ServerClient&) = delete; + ServerClient& operator=(const ServerClient&) = delete; virtual ~ServerClient() = default; // these return true if any messages have been queued for later processing @@ -81,7 +83,7 @@ class ServerClient { SetPeriodicFunc m_setPeriodic; // TODO: make this per-topic? uint32_t m_periodMs{UINT32_MAX}; - ServerImpl& m_server; + ServerStorage& m_storage; int m_id; wpi::Logger& m_logger; diff --git a/ntcore/src/main/native/cpp/server/ServerClient3.cpp b/ntcore/src/main/native/cpp/server/ServerClient3.cpp index ff47581834d..cb0003f5290 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient3.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient3.cpp @@ -111,7 +111,7 @@ void ServerClient3::SendAnnounce(ServerTopic* topic, // subscribe to all non-special topics if (!topic->special) { topic->clients[this].AddSubscriber(m_subscribers[0].get()); - m_server.UpdateMetaTopicSub(topic); + m_storage.UpdateMetaTopicSub(topic); } // NT3 requires a value to send the assign message, so the assign message @@ -240,13 +240,13 @@ void ServerClient3::ClearEntries() { m_publishers.erase(publisherIt); // update meta data - m_server.UpdateMetaTopicPub(topic); + m_storage.UpdateMetaTopicPub(topic); UpdateMetaClientPub(); } } // set retained=false - m_server.SetProperties(this, topic, {{"retained", false}}); + m_storage.SetProperties(this, topic, {{"retained", false}}); } } @@ -275,8 +275,8 @@ void ServerClient3::ClientHello(std::string_view self_id, m_connected = nullptr; // no longer required // create client meta topics - m_metaPub = m_server.CreateMetaTopic(fmt::format("$clientpub${}", m_name)); - m_metaSub = m_server.CreateMetaTopic(fmt::format("$clientsub${}", m_name)); + m_metaPub = m_storage.CreateMetaTopic(fmt::format("$clientpub${}", m_name)); + m_metaSub = m_storage.CreateMetaTopic(fmt::format("$clientsub${}", m_name)); // subscribe and send initial assignments auto& sub = m_subscribers[0]; @@ -291,22 +291,22 @@ void ServerClient3::ClientHello(std::string_view self_id, { auto out = m_wire.Send(); net3::WireEncodeServerHello(out.stream(), 0, "server"); - for (auto&& topic : m_server.m_topics) { + m_storage.ForEachTopic([&](ServerTopic* topic) { if (topic && !topic->special && topic->IsPublished() && topic->lastValue) { DEBUG4("client {}: initial announce of '{}' (id {})", m_id, topic->name, topic->id); topic->clients[this].AddSubscriber(sub.get()); - m_server.UpdateMetaTopicSub(topic.get()); + m_storage.UpdateMetaTopicSub(topic); - TopicData3* topic3 = GetTopic3(topic.get()); + TopicData3* topic3 = GetTopic3(topic); ++topic3->seqNum; net3::WireEncodeEntryAssign(out.stream(), topic->name, topic->id, topic3->seqNum.value(), topic->lastValue, topic3->flags); topic3->sentAssign = true; } - } + }); net3::WireEncodeServerHelloDone(out.stream()); } Flush(); @@ -342,7 +342,7 @@ void ServerClient3::EntryAssign(std::string_view name, unsigned int id, } // create topic - auto topic = m_server.CreateTopic(this, name, typeStr, properties); + auto topic = m_storage.CreateTopic(this, name, typeStr, properties); TopicData3* topic3 = GetTopic3(topic); if (topic3->published || topic3->sentAssign) { WARN("ignoring client {} duplicate publish of '{}'", m_id, name); @@ -365,12 +365,12 @@ void ServerClient3::EntryAssign(std::string_view name, unsigned int id, topic->AddPublisher(this, publisherIt->getSecond().get()); // update meta data - m_server.UpdateMetaTopicPub(topic); + m_storage.UpdateMetaTopicPub(topic); UpdateMetaClientPub(); // acts as an announce + data update SendAnnounce(topic, topic3->pubuid); - m_server.SetValue(this, topic, value); + m_storage.SetValue(this, topic, value); // respond with assign message with assigned topic ID if (m_local && m_state == kStateRunning) { @@ -391,11 +391,7 @@ void ServerClient3::EntryUpdate(unsigned int id, unsigned int seq_num, return; } - if (id >= m_server.m_topics.size()) { - DEBUG3("ignored EntryUpdate from {} on non-existent topic {}", m_id, id); - return; - } - ServerTopic* topic = m_server.m_topics[id].get(); + ServerTopic* topic = m_storage.GetTopic(id); if (!topic || !topic->IsPublished()) { DEBUG3("ignored EntryUpdate from {} on non-existent topic {}", m_id, id); return; @@ -415,13 +411,13 @@ void ServerClient3::EntryUpdate(unsigned int id, unsigned int seq_num, topic->AddPublisher(this, publisherIt->getSecond().get()); // update meta data - m_server.UpdateMetaTopicPub(topic); + m_storage.UpdateMetaTopicPub(topic); UpdateMetaClientPub(); } } topic3->seqNum = net3::SequenceNumber{seq_num}; - m_server.SetValue(this, topic, value); + m_storage.SetValue(this, topic, value); } void ServerClient3::FlagsUpdate(unsigned int id, unsigned int flags) { @@ -430,11 +426,7 @@ void ServerClient3::FlagsUpdate(unsigned int id, unsigned int flags) { m_decoder.SetError("received unexpected FlagsUpdate message"); return; } - if (id >= m_server.m_topics.size()) { - DEBUG3("ignored FlagsUpdate from {} on non-existent topic {}", m_id, id); - return; - } - ServerTopic* topic = m_server.m_topics[id].get(); + ServerTopic* topic = m_storage.GetTopic(id); if (!topic || !topic->IsPublished()) { DEBUG3("ignored FlagsUpdate from {} on non-existent topic {}", m_id, id); return; @@ -443,7 +435,7 @@ void ServerClient3::FlagsUpdate(unsigned int id, unsigned int flags) { DEBUG3("ignored FlagsUpdate from {} on special topic {}", m_id, id); return; } - m_server.SetFlags(this, topic, flags); + m_storage.SetFlags(this, topic, flags); } void ServerClient3::EntryDelete(unsigned int id) { @@ -452,11 +444,7 @@ void ServerClient3::EntryDelete(unsigned int id) { m_decoder.SetError("received unexpected EntryDelete message"); return; } - if (id >= m_server.m_topics.size()) { - DEBUG3("ignored EntryDelete from {} on non-existent topic {}", m_id, id); - return; - } - ServerTopic* topic = m_server.m_topics[id].get(); + ServerTopic* topic = m_storage.GetTopic(id); if (!topic || !topic->IsPublished()) { DEBUG3("ignored EntryDelete from {} on non-existent topic {}", m_id, id); return; @@ -483,12 +471,12 @@ void ServerClient3::EntryDelete(unsigned int id) { m_publishers.erase(publisherIt); // update meta data - m_server.UpdateMetaTopicPub(topic); + m_storage.UpdateMetaTopicPub(topic); UpdateMetaClientPub(); } } } // set retained=false - m_server.SetProperties(this, topic, {{"retained", false}}); + m_storage.SetProperties(this, topic, {{"retained", false}}); } diff --git a/ntcore/src/main/native/cpp/server/ServerClient3.h b/ntcore/src/main/native/cpp/server/ServerClient3.h index 7f61c059f3a..190ba6e00f0 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient3.h +++ b/ntcore/src/main/native/cpp/server/ServerClient3.h @@ -21,9 +21,9 @@ class ServerClient3 final : public ServerClient, private net3::MessageHandler3 { public: ServerClient3(std::string_view connInfo, bool local, net3::WireConnection3& wire, Connected3Func connected, - SetPeriodicFunc setPeriodic, ServerImpl& server, int id, + SetPeriodicFunc setPeriodic, ServerStorage& storage, int id, wpi::Logger& logger) - : ServerClient{"", connInfo, local, setPeriodic, server, id, logger}, + : ServerClient{"", connInfo, local, setPeriodic, storage, id, logger}, m_connected{std::move(connected)}, m_wire{wire}, m_decoder{*this}, diff --git a/ntcore/src/main/native/cpp/server/ServerClient4.h b/ntcore/src/main/native/cpp/server/ServerClient4.h index ce3cb906f2f..81fd11c4150 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient4.h +++ b/ntcore/src/main/native/cpp/server/ServerClient4.h @@ -17,9 +17,9 @@ class ServerClient4 final : public ServerClient4Base { public: ServerClient4(std::string_view name, std::string_view connInfo, bool local, net::WireConnection& wire, SetPeriodicFunc setPeriodic, - ServerImpl& server, int id, wpi::Logger& logger) - : ServerClient4Base{name, connInfo, local, setPeriodic, - server, id, logger}, + ServerStorage& storage, int id, wpi::Logger& logger) + : ServerClient4Base{name, connInfo, local, setPeriodic, + storage, id, logger}, m_wire{wire}, m_ping{wire}, m_incoming{logger}, diff --git a/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp b/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp index 601fdf94c37..dd7ab569130 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient4Base.cpp @@ -22,7 +22,7 @@ void ServerClient4Base::ClientPublish(int pubuid, std::string_view name, const wpi::json& properties, const PubSubOptionsImpl& options) { DEBUG3("ClientPublish({}, {}, {}, {})", m_id, name, pubuid, typeStr); - auto topic = m_server.CreateTopic(this, name, typeStr, properties); + auto topic = m_storage.CreateTopic(this, name, typeStr, properties); // create publisher auto [publisherIt, isNew] = m_publishers.try_emplace( @@ -34,7 +34,7 @@ void ServerClient4Base::ClientPublish(int pubuid, std::string_view name, topic->AddPublisher(this, publisherIt->getSecond().get()); // update meta data - m_server.UpdateMetaTopicPub(topic); + m_storage.UpdateMetaTopicPub(topic); } // respond with announce with pubuid to client @@ -58,33 +58,31 @@ void ServerClient4Base::ClientUnpublish(int pubuid) { m_publishers.erase(publisherIt); // update meta data - m_server.UpdateMetaTopicPub(topic); + m_storage.UpdateMetaTopicPub(topic); // delete topic if no longer published if (!topic->IsPublished()) { - m_server.DeleteTopic(topic); + m_storage.DeleteTopic(topic); } } void ServerClient4Base::ClientSetProperties(std::string_view name, const wpi::json& update) { DEBUG4("ClientSetProperties({}, {}, {})", m_id, name, update.dump()); - auto topicIt = m_server.m_nameTopics.find(name); - if (topicIt == m_server.m_nameTopics.end() || - !topicIt->second->IsPublished()) { + ServerTopic* topic = m_storage.GetTopic(name); + if (!topic || !topic->IsPublished()) { WARN( "server ignoring SetProperties({}) from client {} on unpublished topic " "'{}'; publish or set a value first", update.dump(), m_id, name); return; // nothing to do } - auto topic = topicIt->second; if (topic->special) { WARN("server ignoring SetProperties({}) from client {} on meta topic '{}'", update.dump(), m_id, name); return; // nothing to do } - m_server.SetProperties(nullptr, topic, update); + m_storage.SetProperties(nullptr, topic, update); } void ServerClient4Base::ClientSubscribe(int subuid, @@ -115,8 +113,8 @@ void ServerClient4Base::ClientSubscribe(int subuid, // send announcements in first loop and remember what we want to send in // second loop. std::vector dataToSend; - dataToSend.reserve(m_server.m_topics.size()); - for (auto&& topic : m_server.m_topics) { + dataToSend.reserve(m_storage.GetNumTopics()); + m_storage.ForEachTopic([&](ServerTopic* topic) { auto tcdIt = topic->clients.find(this); bool removed = tcdIt != topic->clients.end() && replace && tcdIt->second.subscribers.erase(sub.get()); @@ -138,22 +136,22 @@ void ServerClient4Base::ClientSubscribe(int subuid, } if (added ^ removed) { - UpdatePeriod(tcdIt->second, topic.get()); - m_server.UpdateMetaTopicSub(topic.get()); + UpdatePeriod(tcdIt->second, topic); + m_storage.UpdateMetaTopicSub(topic); } // announce topic to client if not previously announced if (added && !removed && !wasSubscribed) { DEBUG4("client {}: announce {}", m_id, topic->name); - SendAnnounce(topic.get(), std::nullopt); + SendAnnounce(topic, std::nullopt); } // send last value if (added && !sub->GetOptions().topicsOnly && !wasSubscribedValue && topic->lastValue) { - dataToSend.emplace_back(topic.get()); + dataToSend.emplace_back(topic); } - } + }); for (auto topic : dataToSend) { DEBUG4("send last value for {} to client {}", topic->name, m_id); @@ -170,15 +168,15 @@ void ServerClient4Base::ClientUnsubscribe(int subuid) { auto sub = subIt->getSecond().get(); // remove from topics - for (auto&& topic : m_server.m_topics) { + m_storage.ForEachTopic([&](ServerTopic* topic) { auto tcdIt = topic->clients.find(this); if (tcdIt != topic->clients.end()) { if (tcdIt->second.subscribers.erase(sub)) { - UpdatePeriod(tcdIt->second, topic.get()); - m_server.UpdateMetaTopicSub(topic.get()); + UpdatePeriod(tcdIt->second, topic); + m_storage.UpdateMetaTopicSub(topic); } } - } + }); // delete it from client (future value sets will be ignored) m_subscribers.erase(subIt); @@ -199,7 +197,7 @@ void ServerClient4Base::ClientSetValue(int pubuid, const Value& value) { return; // ignore unrecognized pubuids } auto topic = publisherIt->getSecond().get()->GetTopic(); - m_server.SetValue(this, topic, value); + m_storage.SetValue(this, topic, value); } bool ServerClient4Base::DoProcessIncomingMessages( diff --git a/ntcore/src/main/native/cpp/server/ServerClient4Base.h b/ntcore/src/main/native/cpp/server/ServerClient4Base.h index 996fbb2a484..814723095b6 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient4Base.h +++ b/ntcore/src/main/native/cpp/server/ServerClient4Base.h @@ -19,9 +19,9 @@ class ServerClient4Base : public ServerClient, protected net::ClientMessageHandler { public: ServerClient4Base(std::string_view name, std::string_view connInfo, - bool local, SetPeriodicFunc setPeriodic, ServerImpl& server, - int id, wpi::Logger& logger) - : ServerClient{name, connInfo, local, setPeriodic, server, id, logger} {} + bool local, SetPeriodicFunc setPeriodic, + ServerStorage& storage, int id, wpi::Logger& logger) + : ServerClient{name, connInfo, local, setPeriodic, storage, id, logger} {} protected: // ClientMessageHandler interface diff --git a/ntcore/src/main/native/cpp/server/ServerClientLocal.h b/ntcore/src/main/native/cpp/server/ServerClientLocal.h index 0d297cd0b4a..4ddd73365f0 100644 --- a/ntcore/src/main/native/cpp/server/ServerClientLocal.h +++ b/ntcore/src/main/native/cpp/server/ServerClientLocal.h @@ -13,8 +13,8 @@ namespace nt::server { class ServerClientLocal final : public ServerClient4Base { public: - ServerClientLocal(ServerImpl& server, int id, wpi::Logger& logger) - : ServerClient4Base{"", "", true, [](uint32_t) {}, server, id, logger} {} + ServerClientLocal(ServerStorage& storage, int id, wpi::Logger& logger) + : ServerClient4Base{"", "", true, [](uint32_t) {}, storage, id, logger} {} bool ProcessIncomingText(std::string_view data) final { return false; } bool ProcessIncomingBinary(std::span data) final { diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.cpp b/ntcore/src/main/native/cpp/server/ServerImpl.cpp index c34d3d96512..bc73fbde429 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.cpp +++ b/ntcore/src/main/native/cpp/server/ServerImpl.cpp @@ -8,41 +8,30 @@ #include #include -#include #include #include #include -#include -#include #include -#include -#include -#include -#include -#include -#include #include "Log.h" -#include "net/WireEncoder.h" -#include "net3/WireConnection3.h" -#include "net3/WireEncoder3.h" -#include "networktables/NetworkTableValue.h" -#include "ntcore_c.h" #include "server/MessagePackWriter.h" #include "server/ServerClient3.h" #include "server/ServerClient4.h" #include "server/ServerClientLocal.h" -#include "server/ServerPublisher.h" -#include "server/ServerTopic.h" using namespace nt; using namespace nt::server; using namespace mpack; -ServerImpl::ServerImpl(wpi::Logger& logger) : m_logger{logger} { +ServerImpl::ServerImpl(wpi::Logger& logger) + : m_logger{logger}, + m_storage{logger, [this](ServerTopic* topic, ServerClient* client) { + SendAnnounce(topic, client); + }} { // local is client 0 - m_clients.emplace_back(std::make_unique(*this, 0, logger)); + m_clients.emplace_back( + std::make_unique(m_storage, 0, logger)); m_localClient = static_cast(m_clients.back().get()); } @@ -72,14 +61,14 @@ std::pair ServerImpl::AddClient(std::string_view name, auto& clientData = m_clients[index]; clientData = std::make_unique(dedupName, connInfo, local, wire, - std::move(setPeriodic), *this, - index, m_logger); + std::move(setPeriodic), + m_storage, index, m_logger); // create client meta topics clientData->m_metaPub = - CreateMetaTopic(fmt::format("$clientpub${}", dedupName)); + m_storage.CreateMetaTopic(fmt::format("$clientpub${}", dedupName)); clientData->m_metaSub = - CreateMetaTopic(fmt::format("$clientsub${}", dedupName)); + m_storage.CreateMetaTopic(fmt::format("$clientsub${}", dedupName)); // update meta topics clientData->UpdateMetaClientPub(); @@ -108,7 +97,7 @@ int ServerImpl::AddClient3(std::string_view connInfo, bool local, m_clients[index] = std::make_unique( connInfo, local, wire, std::move(connected), std::move(setPeriodic), - *this, index, m_logger); + m_storage, index, m_logger); DEBUG3("AddClient3('{}') -> {}", connInfo, index); return index; @@ -117,572 +106,45 @@ int ServerImpl::AddClient3(std::string_view connInfo, bool local, std::shared_ptr ServerImpl::RemoveClient(int clientId) { DEBUG3("RemoveClient({})", clientId); auto& client = m_clients[clientId]; - - // remove all publishers and subscribers for this client - wpi::SmallVector toDelete; - for (auto&& topic : m_topics) { - bool pubChanged = false; - bool subChanged = false; - auto tcdIt = topic->clients.find(client.get()); - if (tcdIt != topic->clients.end()) { - pubChanged = !tcdIt->second.publishers.empty(); - subChanged = !tcdIt->second.subscribers.empty(); - topic->publisherCount -= tcdIt->second.publishers.size(); - topic->clients.erase(tcdIt); - } - - if (!topic->IsPublished()) { - toDelete.push_back(topic.get()); - } else { - if (pubChanged) { - UpdateMetaTopicPub(topic.get()); - } - if (subChanged) { - UpdateMetaTopicSub(topic.get()); - } - } - } - - // delete unpublished topics - for (auto topic : toDelete) { - DeleteTopic(topic); + if (client) { + m_storage.RemoveClient(client.get()); } - DeleteTopic(client->m_metaPub); - DeleteTopic(client->m_metaSub); - return std::move(client); } -bool ServerImpl::PersistentChanged() { - bool rv = m_persistentChanged; - m_persistentChanged = false; - return rv; -} - -static void DumpValue(wpi::raw_ostream& os, const Value& value, - wpi::json::serializer& s) { - switch (value.type()) { - case NT_BOOLEAN: - if (value.GetBoolean()) { - os << "true"; - } else { - os << "false"; - } - break; - case NT_DOUBLE: - s.dump_float(value.GetDouble()); - break; - case NT_FLOAT: - s.dump_float(value.GetFloat()); - break; - case NT_INTEGER: - s.dump_integer(value.GetInteger()); - break; - case NT_STRING: - os << '"'; - s.dump_escaped(value.GetString(), false); - os << '"'; - break; - case NT_RAW: - case NT_RPC: - os << '"'; - wpi::Base64Encode(os, value.GetRaw()); - os << '"'; - break; - case NT_BOOLEAN_ARRAY: { - os << '['; - bool first = true; - for (auto v : value.GetBooleanArray()) { - if (first) { - first = false; - } else { - os << ", "; - } - if (v) { - os << "true"; - } else { - os << "false"; - } - } - os << ']'; - break; - } - case NT_DOUBLE_ARRAY: { - os << '['; - bool first = true; - for (auto v : value.GetDoubleArray()) { - if (first) { - first = false; - } else { - os << ", "; - } - s.dump_float(v); - } - os << ']'; - break; - } - case NT_FLOAT_ARRAY: { - os << '['; - bool first = true; - for (auto v : value.GetFloatArray()) { - if (first) { - first = false; - } else { - os << ", "; - } - s.dump_float(v); - } - os << ']'; - break; - } - case NT_INTEGER_ARRAY: { - os << '['; - bool first = true; - for (auto v : value.GetIntegerArray()) { - if (first) { - first = false; - } else { - os << ", "; - } - s.dump_integer(v); - } - os << ']'; - break; - } - case NT_STRING_ARRAY: { - os << '['; - bool first = true; - for (auto&& v : value.GetStringArray()) { - if (first) { - first = false; - } else { - os << ", "; - } - os << '"'; - s.dump_escaped(v, false); - os << '"'; - } - os << ']'; - break; - } - default: - os << "null"; - break; - } -} - -void ServerImpl::DumpPersistent(wpi::raw_ostream& os) { - wpi::json::serializer s{os, ' ', 16}; - os << "[\n"; - bool first = true; - for (const auto& topic : m_topics) { - if (!topic->persistent || !topic->lastValue) { +void ServerImpl::SendAnnounce(ServerTopic* topic, ServerClient* client) { + for (auto&& aClient : m_clients) { + if (!aClient) { continue; } - if (first) { - first = false; - } else { - os << ",\n"; - } - os << " {\n \"name\": \""; - s.dump_escaped(topic->name, false); - os << "\",\n \"type\": \""; - s.dump_escaped(topic->typeStr, false); - os << "\",\n \"value\": "; - DumpValue(os, topic->lastValue, s); - os << ",\n \"properties\": "; - s.dump(topic->properties, true, false, 2, 4); - os << "\n }"; - } - os << "\n]\n"; -} - -static std::string* ObjGetString(wpi::json::object_t& obj, std::string_view key, - std::string* error) { - auto it = obj.find(key); - if (it == obj.end()) { - *error = fmt::format("no {} key", key); - return nullptr; - } - auto val = it->second.get_ptr(); - if (!val) { - *error = fmt::format("{} must be a string", key); - } - return val; -} - -std::string ServerImpl::LoadPersistent(std::string_view in) { - if (in.empty()) { - return {}; - } - - wpi::json j; - try { - j = wpi::json::parse(in); - } catch (wpi::json::parse_error& err) { - return fmt::format("could not decode JSON: {}", err.what()); - } - - if (!j.is_array()) { - return "expected JSON array at top level"; - } - bool persistentChanged = m_persistentChanged; - - std::string allerrors; - int i = -1; - auto time = nt::Now(); - for (auto&& jitem : j) { - ++i; - std::string error; - { - auto obj = jitem.get_ptr(); - if (!obj) { - error = "expected item to be an object"; - goto err; - } - - // name - auto name = ObjGetString(*obj, "name", &error); - if (!name) { - goto err; - } - - // type - auto typeStr = ObjGetString(*obj, "type", &error); - if (!typeStr) { - goto err; - } - - // properties - auto propsIt = obj->find("properties"); - if (propsIt == obj->end()) { - error = "no properties key"; - goto err; - } - auto& props = propsIt->second; - if (!props.is_object()) { - error = "properties must be an object"; - goto err; - } - - // check to make sure persistent property is set - auto persistentIt = props.find("persistent"); - if (persistentIt == props.end()) { - error = "no persistent property"; - goto err; - } - if (auto v = persistentIt->get_ptr()) { - if (!*v) { - error = "persistent property is false"; - goto err; - } - } else { - error = "persistent property is not boolean"; - goto err; - } - - // value - auto valueIt = obj->find("value"); - if (valueIt == obj->end()) { - error = "no value key"; - goto err; - } - Value value; - if (*typeStr == "boolean") { - if (auto v = valueIt->second.get_ptr()) { - value = Value::MakeBoolean(*v, time); - } else { - error = "value type mismatch, expected boolean"; - goto err; - } - } else if (*typeStr == "int") { - if (auto v = valueIt->second.get_ptr()) { - value = Value::MakeInteger(*v, time); - } else if (auto v = valueIt->second.get_ptr()) { - value = Value::MakeInteger(*v, time); - } else { - error = "value type mismatch, expected int"; - goto err; - } - } else if (*typeStr == "float") { - if (auto v = valueIt->second.get_ptr()) { - value = Value::MakeFloat(*v, time); - } else { - error = "value type mismatch, expected float"; - goto err; - } - } else if (*typeStr == "double") { - if (auto v = valueIt->second.get_ptr()) { - value = Value::MakeDouble(*v, time); - } else { - error = "value type mismatch, expected double"; - goto err; - } - } else if (*typeStr == "string" || *typeStr == "json") { - if (auto v = valueIt->second.get_ptr()) { - value = Value::MakeString(*v, time); - } else { - error = "value type mismatch, expected string"; - goto err; - } - } else if (*typeStr == "boolean[]") { - auto arr = valueIt->second.get_ptr(); - if (!arr) { - error = "value type mismatch, expected array"; - goto err; - } - std::vector elems; - for (auto&& jelem : valueIt->second) { - if (auto v = jelem.get_ptr()) { - elems.push_back(*v); - } else { - error = "value type mismatch, expected boolean"; - } - } - value = Value::MakeBooleanArray(elems, time); - } else if (*typeStr == "int[]") { - auto arr = valueIt->second.get_ptr(); - if (!arr) { - error = "value type mismatch, expected array"; - goto err; - } - std::vector elems; - for (auto&& jelem : valueIt->second) { - if (auto v = jelem.get_ptr()) { - elems.push_back(*v); - } else if (auto v = jelem.get_ptr()) { - elems.push_back(*v); - } else { - error = "value type mismatch, expected int"; - } - } - value = Value::MakeIntegerArray(elems, time); - } else if (*typeStr == "double[]") { - auto arr = valueIt->second.get_ptr(); - if (!arr) { - error = "value type mismatch, expected array"; - goto err; - } - std::vector elems; - for (auto&& jelem : valueIt->second) { - if (auto v = jelem.get_ptr()) { - elems.push_back(*v); - } else { - error = "value type mismatch, expected double"; - } - } - value = Value::MakeDoubleArray(elems, time); - } else if (*typeStr == "float[]") { - auto arr = valueIt->second.get_ptr(); - if (!arr) { - error = "value type mismatch, expected array"; - goto err; - } - std::vector elems; - for (auto&& jelem : valueIt->second) { - if (auto v = jelem.get_ptr()) { - elems.push_back(*v); - } else { - error = "value type mismatch, expected float"; - } - } - value = Value::MakeFloatArray(elems, time); - } else if (*typeStr == "string[]") { - auto arr = valueIt->second.get_ptr(); - if (!arr) { - error = "value type mismatch, expected array"; - goto err; - } - std::vector elems; - for (auto&& jelem : valueIt->second) { - if (auto v = jelem.get_ptr()) { - elems.emplace_back(*v); - } else { - error = "value type mismatch, expected string"; - } - } - value = Value::MakeStringArray(std::move(elems), time); - } else { - // raw - if (auto v = valueIt->second.get_ptr()) { - std::vector data; - wpi::Base64Decode(*v, &data); - value = Value::MakeRaw(std::move(data), time); - } else { - error = "value type mismatch, expected string"; - goto err; - } - } - - // create persistent topic - auto topic = CreateTopic(nullptr, *name, *typeStr, props); - - // set value - SetValue(nullptr, topic, value); + // look for subscriber matching prefixes + wpi::SmallVector subscribersBuf; + auto subscribers = + aClient->GetSubscribers(topic->name, topic->special, subscribersBuf); + // don't announce to this client if no subscribers + if (subscribers.empty()) { continue; } - err: - allerrors += fmt::format("{}: {}\n", i, error); - } - m_persistentChanged = persistentChanged; // restore flag - - return allerrors; -} - -ServerTopic* ServerImpl::CreateTopic(ServerClient* client, - std::string_view name, - std::string_view typeStr, - const wpi::json& properties, - bool special) { - auto& topic = m_nameTopics[name]; - if (topic) { - if (typeStr != topic->typeStr) { - if (client) { - WARN("client {} publish '{}' conflicting type '{}' (currently '{}')", - client->GetName(), name, typeStr, topic->typeStr); + auto& tcd = topic->clients[aClient.get()]; + bool added = false; + for (auto subscriber : subscribers) { + if (tcd.AddSubscriber(subscriber)) { + added = true; } } - } else { - // new topic - unsigned int id = m_topics.emplace_back( - std::make_unique(m_logger, name, typeStr, properties)); - topic = m_topics[id].get(); - topic->id = id; - topic->special = special; - - for (auto&& aClient : m_clients) { - if (!aClient) { - continue; - } - - // look for subscriber matching prefixes - wpi::SmallVector subscribersBuf; - auto subscribers = - aClient->GetSubscribers(name, topic->special, subscribersBuf); - - // don't announce to this client if no subscribers - if (subscribers.empty()) { - continue; - } - - auto& tcd = topic->clients[aClient.get()]; - bool added = false; - for (auto subscriber : subscribers) { - if (tcd.AddSubscriber(subscriber)) { - added = true; - } - } - if (added) { - aClient->UpdatePeriod(tcd, topic); - } - - if (aClient.get() == client) { - continue; // don't announce to requesting client again - } - - DEBUG4("client {}: announce {}", aClient->GetId(), topic->name); - aClient->SendAnnounce(topic, std::nullopt); - } - - // create meta topics; don't create if topic is itself a meta topic - if (!special) { - topic->metaPub = CreateMetaTopic(fmt::format("$pub${}", name)); - topic->metaSub = CreateMetaTopic(fmt::format("$sub${}", name)); - UpdateMetaTopicPub(topic); - UpdateMetaTopicSub(topic); + if (added) { + aClient->UpdatePeriod(tcd, topic); } - } - return topic; -} - -ServerTopic* ServerImpl::CreateMetaTopic(std::string_view name) { - return CreateTopic(nullptr, name, "msgpack", {{"retained", true}}, true); -} - -void ServerImpl::DeleteTopic(ServerTopic* topic) { - if (!topic) { - return; - } - - // delete meta topics - if (topic->metaPub) { - DeleteTopic(topic->metaPub); - } - if (topic->metaSub) { - DeleteTopic(topic->metaSub); - } - - // unannounce to all subscribers - for (auto&& tcd : topic->clients) { - if (!tcd.second.subscribers.empty()) { - tcd.first->UpdatePeriod(tcd.second, topic); - tcd.first->SendUnannounce(topic); + if (aClient.get() == client) { + continue; // don't announce to requesting client again } - } - - // erase the topic - m_nameTopics.erase(topic->name); - m_topics.erase(topic->id); -} -void ServerImpl::SetProperties(ServerClient* client, ServerTopic* topic, - const wpi::json& update) { - DEBUG4("SetProperties({}, {}, {})", client ? client->GetId() : -1, - topic->name, update.dump()); - bool wasPersistent = topic->persistent; - if (topic->SetProperties(update)) { - // update persistentChanged flag - if (topic->persistent != wasPersistent) { - m_persistentChanged = true; - } - PropertiesChanged(client, topic, update); - } -} - -void ServerImpl::SetFlags(ServerClient* client, ServerTopic* topic, - unsigned int flags) { - bool wasPersistent = topic->persistent; - if (topic->SetFlags(flags)) { - // update persistentChanged flag - if (topic->persistent != wasPersistent) { - m_persistentChanged = true; - wpi::json update; - if (topic->persistent) { - update = {{"persistent", true}}; - } else { - update = {{"persistent", wpi::json::object()}}; - } - PropertiesChanged(client, topic, update); - } - } -} - -void ServerImpl::SetValue(ServerClient* client, ServerTopic* topic, - const Value& value) { - // update retained value if from same client or timestamp newer - if (topic->cached && (!topic->lastValue || topic->lastValueClient == client || - topic->lastValue.time() == 0 || - value.time() >= topic->lastValue.time())) { - DEBUG4("updating '{}' last value (time was {} is {})", topic->name, - topic->lastValue.time(), value.time()); - topic->lastValue = value; - topic->lastValueClient = client; - - // if persistent, update flag - if (topic->persistent) { - m_persistentChanged = true; - } - } - - for (auto&& tcd : topic->clients) { - if (tcd.first != client && - tcd.second.sendMode != net::ValueSendMode::kDisabled) { - tcd.first->SendValue(topic, value, tcd.second.sendMode); - } + DEBUG4("client {}: announce {}", aClient->GetId(), topic->name); + aClient->SendAnnounce(topic, std::nullopt); } } @@ -701,67 +163,13 @@ void ServerImpl::UpdateMetaClients(const std::vector& conns) { } mpack_finish_array(&w); if (mpack_writer_destroy(&w) == mpack_ok) { - SetValue(nullptr, m_metaClients, Value::MakeRaw(std::move(w.bytes))); + m_storage.SetValue(nullptr, m_metaClients, + Value::MakeRaw(std::move(w.bytes))); } else { DEBUG4("failed to encode $clients"); } } -void ServerImpl::UpdateMetaTopicPub(ServerTopic* topic) { - if (!topic->metaPub) { - return; - } - Writer w; - uint32_t count = 0; - for (auto&& tcd : topic->clients) { - count += tcd.second.publishers.size(); - } - mpack_start_array(&w, count); - for (auto&& tcd : topic->clients) { - for (auto&& pub : tcd.second.publishers) { - mpack_write_object_bytes(&w, pub->GetMetaTopicData()); - } - } - mpack_finish_array(&w); - if (mpack_writer_destroy(&w) == mpack_ok) { - SetValue(nullptr, topic->metaPub, Value::MakeRaw(std::move(w.bytes))); - } -} - -void ServerImpl::UpdateMetaTopicSub(ServerTopic* topic) { - if (!topic->metaSub) { - return; - } - Writer w; - uint32_t count = 0; - for (auto&& tcd : topic->clients) { - count += tcd.second.subscribers.size(); - } - mpack_start_array(&w, count); - for (auto&& tcd : topic->clients) { - for (auto&& sub : tcd.second.subscribers) { - mpack_write_object_bytes(&w, sub->GetMetaTopicData()); - } - } - mpack_finish_array(&w); - if (mpack_writer_destroy(&w) == mpack_ok) { - SetValue(nullptr, topic->metaSub, Value::MakeRaw(std::move(w.bytes))); - } -} - -void ServerImpl::PropertiesChanged(ServerClient* client, ServerTopic* topic, - const wpi::json& update) { - // removing some properties can result in the topic being unpublished - if (!topic->IsPublished()) { - DeleteTopic(topic); - } else { - // send updated announcement to all subscribers - for (auto&& tcd : topic->clients) { - tcd.first->SendPropertiesUpdate(topic, update, tcd.first == client); - } - } -} - void ServerImpl::SendAllOutgoing(uint64_t curTimeMs, bool flush) { for (auto&& client : m_clients) { if (client) { @@ -782,11 +190,11 @@ void ServerImpl::SetLocal(net::ServerMessageHandler* local, m_localClient->SetLocal(local, queue); // create server meta topics - m_metaClients = CreateMetaTopic("$clients"); + m_metaClients = m_storage.CreateMetaTopic("$clients"); // create local client meta topics - m_localClient->m_metaPub = CreateMetaTopic("$serverpub"); - m_localClient->m_metaSub = CreateMetaTopic("$serversub"); + m_localClient->m_metaPub = m_storage.CreateMetaTopic("$serverpub"); + m_localClient->m_metaSub = m_storage.CreateMetaTopic("$serversub"); // update meta topics m_localClient->UpdateMetaClientPub(); @@ -833,7 +241,7 @@ void ServerImpl::ConnectionsChanged(const std::vector& conns) { std::string ServerImpl::DumpPersistent() { std::string rv; wpi::raw_string_ostream os{rv}; - DumpPersistent(os); + m_storage.DumpPersistent(os); os.flush(); return rv; } diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.h b/ntcore/src/main/native/cpp/server/ServerImpl.h index 65a97d50333..492d3e0a56f 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.h +++ b/ntcore/src/main/native/cpp/server/ServerImpl.h @@ -13,13 +13,9 @@ #include #include -#include -#include -#include - #include "server/Functions.h" #include "server/ServerClient.h" -#include "server/ServerTopic.h" +#include "server/ServerStorage.h" namespace wpi { class Logger; @@ -41,18 +37,10 @@ class WireConnection3; namespace nt::server { -class ServerClient; -class ServerClient3; -class ServerClient4; -class ServerClient4Base; class ServerClientLocal; +struct ServerTopic; class ServerImpl final { - friend class ServerClient; - friend class ServerClient3; - friend class ServerClient4; - friend class ServerClient4Base; - public: explicit ServerImpl(wpi::Logger& logger); @@ -84,44 +72,28 @@ class ServerImpl final { void ConnectionsChanged(const std::vector& conns); // if any persistent values changed since the last call to this function - bool PersistentChanged(); + bool PersistentChanged() { return m_storage.PersistentChanged(); } + std::string DumpPersistent(); // returns newline-separated errors - std::string LoadPersistent(std::string_view in); + std::string LoadPersistent(std::string_view in) { + return m_storage.LoadPersistent(in); + } private: wpi::Logger& m_logger; ServerClientLocal* m_localClient; std::vector> m_clients; - wpi::UidVector, 16> m_topics; - wpi::StringMap m_nameTopics; - bool m_persistentChanged{false}; + + ServerStorage m_storage; // global meta topics (other meta topics are linked to from the specific // client or topic) ServerTopic* m_metaClients; - void DumpPersistent(wpi::raw_ostream& os); - - // helper functions - ServerTopic* CreateTopic(ServerClient* client, std::string_view name, - std::string_view typeStr, - const wpi::json& properties, bool special = false); - ServerTopic* CreateMetaTopic(std::string_view name); - void DeleteTopic(ServerTopic* topic); - void SetProperties(ServerClient* client, ServerTopic* topic, - const wpi::json& update); - void SetFlags(ServerClient* client, ServerTopic* topic, unsigned int flags); - void SetValue(ServerClient* client, ServerTopic* topic, const Value& value); - - // update meta topic values from data structures + void SendAnnounce(ServerTopic* topic, ServerClient* client); void UpdateMetaClients(const std::vector& conns); - void UpdateMetaTopicPub(ServerTopic* topic); - void UpdateMetaTopicSub(ServerTopic* topic); - - void PropertiesChanged(ServerClient* client, ServerTopic* topic, - const wpi::json& update); }; } // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerPublisher.h b/ntcore/src/main/native/cpp/server/ServerPublisher.h index dcfe3dc74c1..e8c0677c8c5 100644 --- a/ntcore/src/main/native/cpp/server/ServerPublisher.h +++ b/ntcore/src/main/native/cpp/server/ServerPublisher.h @@ -23,6 +23,8 @@ class ServerPublisher { : m_clientName{clientName}, m_topic{topic}, m_pubuid{pubuid} { UpdateMeta(); } + ServerPublisher(const ServerPublisher&) = delete; + ServerPublisher& operator=(const ServerPublisher&) = delete; ServerTopic* GetTopic() const { return m_topic; } std::span GetMetaClientData() const { return m_metaClient; } diff --git a/ntcore/src/main/native/cpp/server/ServerStorage.cpp b/ntcore/src/main/native/cpp/server/ServerStorage.cpp new file mode 100644 index 00000000000..31fbe95bd11 --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerStorage.cpp @@ -0,0 +1,603 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "ServerStorage.h" + +#include +#include +#include +#include + +#include "Log.h" +#include "server/MessagePackWriter.h" +#include "server/ServerClient.h" + +using namespace nt; +using namespace nt::server; +using namespace mpack; + +ServerTopic* ServerStorage::CreateTopic(ServerClient* client, + std::string_view name, + std::string_view typeStr, + const wpi::json& properties, + bool special) { + auto& topic = m_nameTopics[name]; + if (topic) { + if (typeStr != topic->typeStr) { + if (client) { + WARN("client {} publish '{}' conflicting type '{}' (currently '{}')", + client->GetName(), name, typeStr, topic->typeStr); + } + } + } else { + // new topic + unsigned int id = m_topics.emplace_back( + std::make_unique(m_logger, name, typeStr, properties)); + topic = m_topics[id].get(); + topic->id = id; + topic->special = special; + + m_sendAnnounce(topic, client); + + // create meta topics; don't create if topic is itself a meta topic + if (!special) { + topic->metaPub = CreateMetaTopic(fmt::format("$pub${}", name)); + topic->metaSub = CreateMetaTopic(fmt::format("$sub${}", name)); + UpdateMetaTopicPub(topic); + UpdateMetaTopicSub(topic); + } + } + + return topic; +} + +ServerTopic* ServerStorage::CreateMetaTopic(std::string_view name) { + return CreateTopic(nullptr, name, "msgpack", {{"retained", true}}, true); +} + +void ServerStorage::DeleteTopic(ServerTopic* topic) { + if (!topic) { + return; + } + + // delete meta topics + if (topic->metaPub) { + DeleteTopic(topic->metaPub); + } + if (topic->metaSub) { + DeleteTopic(topic->metaSub); + } + + // unannounce to all subscribers + for (auto&& tcd : topic->clients) { + if (!tcd.second.subscribers.empty()) { + tcd.first->UpdatePeriod(tcd.second, topic); + tcd.first->SendUnannounce(topic); + } + } + + // erase the topic + m_nameTopics.erase(topic->name); + m_topics.erase(topic->id); +} + +void ServerStorage::SetProperties(ServerClient* client, ServerTopic* topic, + const wpi::json& update) { + DEBUG4("SetProperties({}, {}, {})", client ? client->GetId() : -1, + topic->name, update.dump()); + bool wasPersistent = topic->persistent; + if (topic->SetProperties(update)) { + // update persistentChanged flag + if (topic->persistent != wasPersistent) { + m_persistentChanged = true; + } + PropertiesChanged(client, topic, update); + } +} + +void ServerStorage::SetFlags(ServerClient* client, ServerTopic* topic, + unsigned int flags) { + bool wasPersistent = topic->persistent; + if (topic->SetFlags(flags)) { + // update persistentChanged flag + if (topic->persistent != wasPersistent) { + m_persistentChanged = true; + wpi::json update; + if (topic->persistent) { + update = {{"persistent", true}}; + } else { + update = {{"persistent", wpi::json::object()}}; + } + PropertiesChanged(client, topic, update); + } + } +} + +void ServerStorage::SetValue(ServerClient* client, ServerTopic* topic, + const Value& value) { + // update retained value if from same client or timestamp newer + if (topic->cached && (!topic->lastValue || topic->lastValueClient == client || + topic->lastValue.time() == 0 || + value.time() >= topic->lastValue.time())) { + DEBUG4("updating '{}' last value (time was {} is {})", topic->name, + topic->lastValue.time(), value.time()); + topic->lastValue = value; + topic->lastValueClient = client; + + // if persistent, update flag + if (topic->persistent) { + m_persistentChanged = true; + } + } + + for (auto&& tcd : topic->clients) { + if (tcd.first != client && + tcd.second.sendMode != net::ValueSendMode::kDisabled) { + tcd.first->SendValue(topic, value, tcd.second.sendMode); + } + } +} + +void ServerStorage::RemoveClient(ServerClient* client) { + // remove all publishers and subscribers for this client + wpi::SmallVector toDelete; + for (auto&& topic : m_topics) { + bool pubChanged = false; + bool subChanged = false; + auto tcdIt = topic->clients.find(client); + if (tcdIt != topic->clients.end()) { + pubChanged = !tcdIt->second.publishers.empty(); + subChanged = !tcdIt->second.subscribers.empty(); + topic->publisherCount -= tcdIt->second.publishers.size(); + topic->clients.erase(tcdIt); + } + + if (!topic->IsPublished()) { + toDelete.push_back(topic.get()); + } else { + if (pubChanged) { + UpdateMetaTopicPub(topic.get()); + } + if (subChanged) { + UpdateMetaTopicSub(topic.get()); + } + } + } + + // delete unpublished topics + for (auto topic : toDelete) { + DeleteTopic(topic); + } + + DeleteTopic(client->m_metaPub); + DeleteTopic(client->m_metaSub); +} + +void ServerStorage::UpdateMetaTopicPub(ServerTopic* topic) { + if (!topic->metaPub) { + return; + } + Writer w; + uint32_t count = 0; + for (auto&& tcd : topic->clients) { + count += tcd.second.publishers.size(); + } + mpack_start_array(&w, count); + for (auto&& tcd : topic->clients) { + for (auto&& pub : tcd.second.publishers) { + mpack_write_object_bytes(&w, pub->GetMetaTopicData()); + } + } + mpack_finish_array(&w); + if (mpack_writer_destroy(&w) == mpack_ok) { + SetValue(nullptr, topic->metaPub, Value::MakeRaw(std::move(w.bytes))); + } +} + +void ServerStorage::UpdateMetaTopicSub(ServerTopic* topic) { + if (!topic->metaSub) { + return; + } + Writer w; + uint32_t count = 0; + for (auto&& tcd : topic->clients) { + count += tcd.second.subscribers.size(); + } + mpack_start_array(&w, count); + for (auto&& tcd : topic->clients) { + for (auto&& sub : tcd.second.subscribers) { + mpack_write_object_bytes(&w, sub->GetMetaTopicData()); + } + } + mpack_finish_array(&w); + if (mpack_writer_destroy(&w) == mpack_ok) { + SetValue(nullptr, topic->metaSub, Value::MakeRaw(std::move(w.bytes))); + } +} + +void ServerStorage::PropertiesChanged(ServerClient* client, ServerTopic* topic, + const wpi::json& update) { + // removing some properties can result in the topic being unpublished + if (!topic->IsPublished()) { + DeleteTopic(topic); + } else { + // send updated announcement to all subscribers + for (auto&& tcd : topic->clients) { + tcd.first->SendPropertiesUpdate(topic, update, tcd.first == client); + } + } +} + +static void DumpValue(wpi::raw_ostream& os, const Value& value, + wpi::json::serializer& s) { + switch (value.type()) { + case NT_BOOLEAN: + if (value.GetBoolean()) { + os << "true"; + } else { + os << "false"; + } + break; + case NT_DOUBLE: + s.dump_float(value.GetDouble()); + break; + case NT_FLOAT: + s.dump_float(value.GetFloat()); + break; + case NT_INTEGER: + s.dump_integer(value.GetInteger()); + break; + case NT_STRING: + os << '"'; + s.dump_escaped(value.GetString(), false); + os << '"'; + break; + case NT_RAW: + case NT_RPC: + os << '"'; + wpi::Base64Encode(os, value.GetRaw()); + os << '"'; + break; + case NT_BOOLEAN_ARRAY: { + os << '['; + bool first = true; + for (auto v : value.GetBooleanArray()) { + if (first) { + first = false; + } else { + os << ", "; + } + if (v) { + os << "true"; + } else { + os << "false"; + } + } + os << ']'; + break; + } + case NT_DOUBLE_ARRAY: { + os << '['; + bool first = true; + for (auto v : value.GetDoubleArray()) { + if (first) { + first = false; + } else { + os << ", "; + } + s.dump_float(v); + } + os << ']'; + break; + } + case NT_FLOAT_ARRAY: { + os << '['; + bool first = true; + for (auto v : value.GetFloatArray()) { + if (first) { + first = false; + } else { + os << ", "; + } + s.dump_float(v); + } + os << ']'; + break; + } + case NT_INTEGER_ARRAY: { + os << '['; + bool first = true; + for (auto v : value.GetIntegerArray()) { + if (first) { + first = false; + } else { + os << ", "; + } + s.dump_integer(v); + } + os << ']'; + break; + } + case NT_STRING_ARRAY: { + os << '['; + bool first = true; + for (auto&& v : value.GetStringArray()) { + if (first) { + first = false; + } else { + os << ", "; + } + os << '"'; + s.dump_escaped(v, false); + os << '"'; + } + os << ']'; + break; + } + default: + os << "null"; + break; + } +} + +void ServerStorage::DumpPersistent(wpi::raw_ostream& os) { + wpi::json::serializer s{os, ' ', 16}; + os << "[\n"; + bool first = true; + for (const auto& topic : m_topics) { + if (!topic->persistent || !topic->lastValue) { + continue; + } + if (first) { + first = false; + } else { + os << ",\n"; + } + os << " {\n \"name\": \""; + s.dump_escaped(topic->name, false); + os << "\",\n \"type\": \""; + s.dump_escaped(topic->typeStr, false); + os << "\",\n \"value\": "; + DumpValue(os, topic->lastValue, s); + os << ",\n \"properties\": "; + s.dump(topic->properties, true, false, 2, 4); + os << "\n }"; + } + os << "\n]\n"; +} + +static std::string* ObjGetString(wpi::json::object_t& obj, std::string_view key, + std::string* error) { + auto it = obj.find(key); + if (it == obj.end()) { + *error = fmt::format("no {} key", key); + return nullptr; + } + auto val = it->second.get_ptr(); + if (!val) { + *error = fmt::format("{} must be a string", key); + } + return val; +} + +std::string ServerStorage::LoadPersistent(std::string_view in) { + if (in.empty()) { + return {}; + } + + wpi::json j; + try { + j = wpi::json::parse(in); + } catch (wpi::json::parse_error& err) { + return fmt::format("could not decode JSON: {}", err.what()); + } + + if (!j.is_array()) { + return "expected JSON array at top level"; + } + + bool persistentChanged = m_persistentChanged; + + std::string allerrors; + int i = -1; + auto time = nt::Now(); + for (auto&& jitem : j) { + ++i; + std::string error; + { + auto obj = jitem.get_ptr(); + if (!obj) { + error = "expected item to be an object"; + goto err; + } + + // name + auto name = ObjGetString(*obj, "name", &error); + if (!name) { + goto err; + } + + // type + auto typeStr = ObjGetString(*obj, "type", &error); + if (!typeStr) { + goto err; + } + + // properties + auto propsIt = obj->find("properties"); + if (propsIt == obj->end()) { + error = "no properties key"; + goto err; + } + auto& props = propsIt->second; + if (!props.is_object()) { + error = "properties must be an object"; + goto err; + } + + // check to make sure persistent property is set + auto persistentIt = props.find("persistent"); + if (persistentIt == props.end()) { + error = "no persistent property"; + goto err; + } + if (auto v = persistentIt->get_ptr()) { + if (!*v) { + error = "persistent property is false"; + goto err; + } + } else { + error = "persistent property is not boolean"; + goto err; + } + + // value + auto valueIt = obj->find("value"); + if (valueIt == obj->end()) { + error = "no value key"; + goto err; + } + Value value; + if (*typeStr == "boolean") { + if (auto v = valueIt->second.get_ptr()) { + value = Value::MakeBoolean(*v, time); + } else { + error = "value type mismatch, expected boolean"; + goto err; + } + } else if (*typeStr == "int") { + if (auto v = valueIt->second.get_ptr()) { + value = Value::MakeInteger(*v, time); + } else if (auto v = valueIt->second.get_ptr()) { + value = Value::MakeInteger(*v, time); + } else { + error = "value type mismatch, expected int"; + goto err; + } + } else if (*typeStr == "float") { + if (auto v = valueIt->second.get_ptr()) { + value = Value::MakeFloat(*v, time); + } else { + error = "value type mismatch, expected float"; + goto err; + } + } else if (*typeStr == "double") { + if (auto v = valueIt->second.get_ptr()) { + value = Value::MakeDouble(*v, time); + } else { + error = "value type mismatch, expected double"; + goto err; + } + } else if (*typeStr == "string" || *typeStr == "json") { + if (auto v = valueIt->second.get_ptr()) { + value = Value::MakeString(*v, time); + } else { + error = "value type mismatch, expected string"; + goto err; + } + } else if (*typeStr == "boolean[]") { + auto arr = valueIt->second.get_ptr(); + if (!arr) { + error = "value type mismatch, expected array"; + goto err; + } + std::vector elems; + for (auto&& jelem : valueIt->second) { + if (auto v = jelem.get_ptr()) { + elems.push_back(*v); + } else { + error = "value type mismatch, expected boolean"; + } + } + value = Value::MakeBooleanArray(elems, time); + } else if (*typeStr == "int[]") { + auto arr = valueIt->second.get_ptr(); + if (!arr) { + error = "value type mismatch, expected array"; + goto err; + } + std::vector elems; + for (auto&& jelem : valueIt->second) { + if (auto v = jelem.get_ptr()) { + elems.push_back(*v); + } else if (auto v = jelem.get_ptr()) { + elems.push_back(*v); + } else { + error = "value type mismatch, expected int"; + } + } + value = Value::MakeIntegerArray(elems, time); + } else if (*typeStr == "double[]") { + auto arr = valueIt->second.get_ptr(); + if (!arr) { + error = "value type mismatch, expected array"; + goto err; + } + std::vector elems; + for (auto&& jelem : valueIt->second) { + if (auto v = jelem.get_ptr()) { + elems.push_back(*v); + } else { + error = "value type mismatch, expected double"; + } + } + value = Value::MakeDoubleArray(elems, time); + } else if (*typeStr == "float[]") { + auto arr = valueIt->second.get_ptr(); + if (!arr) { + error = "value type mismatch, expected array"; + goto err; + } + std::vector elems; + for (auto&& jelem : valueIt->second) { + if (auto v = jelem.get_ptr()) { + elems.push_back(*v); + } else { + error = "value type mismatch, expected float"; + } + } + value = Value::MakeFloatArray(elems, time); + } else if (*typeStr == "string[]") { + auto arr = valueIt->second.get_ptr(); + if (!arr) { + error = "value type mismatch, expected array"; + goto err; + } + std::vector elems; + for (auto&& jelem : valueIt->second) { + if (auto v = jelem.get_ptr()) { + elems.emplace_back(*v); + } else { + error = "value type mismatch, expected string"; + } + } + value = Value::MakeStringArray(std::move(elems), time); + } else { + // raw + if (auto v = valueIt->second.get_ptr()) { + std::vector data; + wpi::Base64Decode(*v, &data); + value = Value::MakeRaw(std::move(data), time); + } else { + error = "value type mismatch, expected string"; + goto err; + } + } + + // create persistent topic + auto topic = CreateTopic(nullptr, *name, *typeStr, props); + + // set value + SetValue(nullptr, topic, value); + + continue; + } + err: + allerrors += fmt::format("{}: {}\n", i, error); + } + + m_persistentChanged = persistentChanged; // restore flag + + return allerrors; +} diff --git a/ntcore/src/main/native/cpp/server/ServerStorage.h b/ntcore/src/main/native/cpp/server/ServerStorage.h new file mode 100644 index 00000000000..24d043ae51a --- /dev/null +++ b/ntcore/src/main/native/cpp/server/ServerStorage.h @@ -0,0 +1,90 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include "server/ServerTopic.h" + +namespace wpi { +class Logger; +class raw_ostream; +} // namespace wpi + +namespace nt::server { + +class ServerClient; + +class ServerStorage final { + public: + ServerStorage(wpi::Logger& logger, + std::function + sendAnnounce) + : m_logger{logger}, m_sendAnnounce{std::move(sendAnnounce)} {} + ServerStorage(const ServerStorage&) = delete; + ServerStorage& operator=(const ServerStorage&) = delete; + + ServerTopic* CreateTopic(ServerClient* client, std::string_view name, + std::string_view typeStr, + const wpi::json& properties, bool special = false); + ServerTopic* CreateMetaTopic(std::string_view name); + void DeleteTopic(ServerTopic* topic); + void SetProperties(ServerClient* client, ServerTopic* topic, + const wpi::json& update); + void SetFlags(ServerClient* client, ServerTopic* topic, unsigned int flags); + void SetValue(ServerClient* client, ServerTopic* topic, const Value& value); + + void RemoveClient(ServerClient* client); + + void PropertiesChanged(ServerClient* client, ServerTopic* topic, + const wpi::json& update); + + ServerTopic* GetTopic(unsigned int id) const { + return id < m_topics.size() ? m_topics[id].get() : nullptr; + } + ServerTopic* GetTopic(std::string_view name) const { + return m_nameTopics.lookup(name); + } + + // Approximate upper bound, not exact quantity + size_t GetNumTopics() const { return m_topics.size(); } + + void ForEachTopic(std::invocable auto&& func) const { + for (auto&& topic : m_topics) { + func(topic.get()); + } + } + + // update meta topic values from data structures + void UpdateMetaTopicPub(ServerTopic* topic); + void UpdateMetaTopicSub(ServerTopic* topic); + + // if any persistent values changed since the last call to this function + bool PersistentChanged() { + bool rv = m_persistentChanged; + m_persistentChanged = false; + return rv; + } + + void DumpPersistent(wpi::raw_ostream& os); + // returns newline-separated errors + std::string LoadPersistent(std::string_view in); + + private: + wpi::Logger& m_logger; + std::function m_sendAnnounce; + + wpi::UidVector, 16> m_topics; + wpi::StringMap m_nameTopics; + bool m_persistentChanged{false}; +}; + +} // namespace nt::server diff --git a/ntcore/src/main/native/cpp/server/ServerSubscriber.h b/ntcore/src/main/native/cpp/server/ServerSubscriber.h index d59b7590193..47a79eb3bdb 100644 --- a/ntcore/src/main/native/cpp/server/ServerSubscriber.h +++ b/ntcore/src/main/native/cpp/server/ServerSubscriber.h @@ -35,6 +35,8 @@ class ServerSubscriber { m_periodMs = kMinPeriodMs; } } + ServerSubscriber(const ServerSubscriber&) = delete; + ServerSubscriber& operator=(const ServerSubscriber&) = delete; void Update(std::span topicNames_, const PubSubOptionsImpl& options_) { diff --git a/ntcore/src/main/native/cpp/server/ServerTopic.h b/ntcore/src/main/native/cpp/server/ServerTopic.h index d1ca5c77dc6..82589726da5 100644 --- a/ntcore/src/main/native/cpp/server/ServerTopic.h +++ b/ntcore/src/main/native/cpp/server/ServerTopic.h @@ -57,6 +57,8 @@ struct ServerTopic { properties(std::move(properties)) { RefreshProperties(); } + ServerTopic(const ServerTopic&) = delete; + ServerTopic& operator=(const ServerTopic&) = delete; bool IsPublished() const { return persistent || retained || publisherCount != 0; From 9c2c8fef066c82b5d8a8efade0356156e4bbb54b Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Wed, 16 Oct 2024 23:10:52 -0700 Subject: [PATCH 11/26] [ntcore] ServerImpl: Move some creation to ServerClient4 --- .../main/native/cpp/server/ServerClient4.cpp | 21 +++++++++++++++++++ .../main/native/cpp/server/ServerClient4.h | 8 +------ .../src/main/native/cpp/server/ServerImpl.cpp | 10 --------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/ntcore/src/main/native/cpp/server/ServerClient4.cpp b/ntcore/src/main/native/cpp/server/ServerClient4.cpp index f96844f8c6f..9b6299fded3 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient4.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClient4.cpp @@ -10,10 +10,31 @@ #include "Log.h" #include "net/WireDecoder.h" +#include "server/ServerStorage.h" #include "server/ServerTopic.h" using namespace nt::server; +ServerClient4::ServerClient4(std::string_view name, std::string_view connInfo, + bool local, net::WireConnection& wire, + SetPeriodicFunc setPeriodic, + ServerStorage& storage, int id, + wpi::Logger& logger) + : ServerClient4Base{name, connInfo, local, setPeriodic, + storage, id, logger}, + m_wire{wire}, + m_ping{wire}, + m_incoming{logger}, + m_outgoing{wire, local} { + // create client meta topics + m_metaPub = storage.CreateMetaTopic(fmt::format("$clientpub${}", name)); + m_metaSub = storage.CreateMetaTopic(fmt::format("$clientsub${}", name)); + + // update meta topics + UpdateMetaClientPub(); + UpdateMetaClientSub(); +} + bool ServerClient4::ProcessIncomingText(std::string_view data) { constexpr int kMaxImmProcessing = 10; bool queueWasEmpty = m_incoming.empty(); diff --git a/ntcore/src/main/native/cpp/server/ServerClient4.h b/ntcore/src/main/native/cpp/server/ServerClient4.h index 81fd11c4150..7c58addf019 100644 --- a/ntcore/src/main/native/cpp/server/ServerClient4.h +++ b/ntcore/src/main/native/cpp/server/ServerClient4.h @@ -17,13 +17,7 @@ class ServerClient4 final : public ServerClient4Base { public: ServerClient4(std::string_view name, std::string_view connInfo, bool local, net::WireConnection& wire, SetPeriodicFunc setPeriodic, - ServerStorage& storage, int id, wpi::Logger& logger) - : ServerClient4Base{name, connInfo, local, setPeriodic, - storage, id, logger}, - m_wire{wire}, - m_ping{wire}, - m_incoming{logger}, - m_outgoing{wire, local} {} + ServerStorage& storage, int id, wpi::Logger& logger); bool ProcessIncomingText(std::string_view data) final; bool ProcessIncomingBinary(std::span data) final; diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.cpp b/ntcore/src/main/native/cpp/server/ServerImpl.cpp index bc73fbde429..5aa2f4714c4 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.cpp +++ b/ntcore/src/main/native/cpp/server/ServerImpl.cpp @@ -64,16 +64,6 @@ std::pair ServerImpl::AddClient(std::string_view name, std::move(setPeriodic), m_storage, index, m_logger); - // create client meta topics - clientData->m_metaPub = - m_storage.CreateMetaTopic(fmt::format("$clientpub${}", dedupName)); - clientData->m_metaSub = - m_storage.CreateMetaTopic(fmt::format("$clientsub${}", dedupName)); - - // update meta topics - clientData->UpdateMetaClientPub(); - clientData->UpdateMetaClientSub(); - DEBUG3("AddClient('{}', '{}') -> {}", name, connInfo, index); return {std::move(dedupName), index}; } From 98a1a4bbdfbbf15952ca555a98485dc68f76266b Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Wed, 16 Oct 2024 23:14:19 -0700 Subject: [PATCH 12/26] [ntcore] ServerImpl: Refactor getting empty client slot --- .../src/main/native/cpp/server/ServerImpl.cpp | 39 +++++++------------ .../src/main/native/cpp/server/ServerImpl.h | 1 + 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.cpp b/ntcore/src/main/native/cpp/server/ServerImpl.cpp index 5aa2f4714c4..929c72c8e04 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.cpp +++ b/ntcore/src/main/native/cpp/server/ServerImpl.cpp @@ -43,18 +43,7 @@ std::pair ServerImpl::AddClient(std::string_view name, if (name.empty()) { name = "NT4"; } - size_t index = m_clients.size(); - // find an empty slot - // just do a linear search as number of clients is typically small (<10) - for (size_t i = 0, end = index; i < end; ++i) { - if (!m_clients[i]) { - index = i; - break; - } - } - if (index == m_clients.size()) { - m_clients.emplace_back(); - } + size_t index = GetEmptyClientSlot(); // ensure name is unique by suffixing index std::string dedupName = fmt::format("{}@{}", name, index); @@ -72,18 +61,7 @@ int ServerImpl::AddClient3(std::string_view connInfo, bool local, net3::WireConnection3& wire, Connected3Func connected, SetPeriodicFunc setPeriodic) { - size_t index = m_clients.size(); - // find an empty slot; we can't check for duplicates until we get a hello. - // just do a linear search as number of clients is typically small (<10) - for (size_t i = 0, end = index; i < end; ++i) { - if (!m_clients[i]) { - index = i; - break; - } - } - if (index == m_clients.size()) { - m_clients.emplace_back(); - } + size_t index = GetEmptyClientSlot(); m_clients[index] = std::make_unique( connInfo, local, wire, std::move(connected), std::move(setPeriodic), @@ -102,6 +80,19 @@ std::shared_ptr ServerImpl::RemoveClient(int clientId) { return std::move(client); } +size_t ServerImpl::GetEmptyClientSlot() { + size_t size = m_clients.size(); + // find an empty slot + // just do a linear search as number of clients is typically small (<10) + for (size_t i = 0, end = size; i < end; ++i) { + if (!m_clients[i]) { + return i; + } + } + m_clients.emplace_back(); + return size; +} + void ServerImpl::SendAnnounce(ServerTopic* topic, ServerClient* client) { for (auto&& aClient : m_clients) { if (!aClient) { diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.h b/ntcore/src/main/native/cpp/server/ServerImpl.h index 492d3e0a56f..d461ce7f3b8 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.h +++ b/ntcore/src/main/native/cpp/server/ServerImpl.h @@ -92,6 +92,7 @@ class ServerImpl final { // client or topic) ServerTopic* m_metaClients; + size_t GetEmptyClientSlot(); void SendAnnounce(ServerTopic* topic, ServerClient* client); void UpdateMetaClients(const std::vector& conns); }; From fc437ece906c145dcb84b26e6f2814d646e76572 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Wed, 16 Oct 2024 23:17:44 -0700 Subject: [PATCH 13/26] [ntcore] ServerImpl: Inlining --- ntcore/src/main/native/cpp/server/ServerImpl.cpp | 4 ---- ntcore/src/main/native/cpp/server/ServerImpl.h | 4 +++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.cpp b/ntcore/src/main/native/cpp/server/ServerImpl.cpp index 929c72c8e04..8265ac71fda 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.cpp +++ b/ntcore/src/main/native/cpp/server/ServerImpl.cpp @@ -215,10 +215,6 @@ bool ServerImpl::ProcessLocalMessages(size_t max) { return m_localClient->ProcessIncomingMessages(max); } -void ServerImpl::ConnectionsChanged(const std::vector& conns) { - UpdateMetaClients(conns); -} - std::string ServerImpl::DumpPersistent() { std::string rv; wpi::raw_string_ostream os{rv}; diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.h b/ntcore/src/main/native/cpp/server/ServerImpl.h index d461ce7f3b8..f78ee672d3b 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.h +++ b/ntcore/src/main/native/cpp/server/ServerImpl.h @@ -69,7 +69,9 @@ class ServerImpl final { SetPeriodicFunc setPeriodic); std::shared_ptr RemoveClient(int clientId); - void ConnectionsChanged(const std::vector& conns); + void ConnectionsChanged(const std::vector& conns) { + UpdateMetaClients(conns); + } // if any persistent values changed since the last call to this function bool PersistentChanged() { return m_storage.PersistentChanged(); } From 03ac7a352fbe90be2995b48374502204d8cb34bb Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Wed, 16 Oct 2024 23:35:52 -0700 Subject: [PATCH 14/26] [ntcore] Move metatopic creation from SetLocal to constructor --- .../main/native/cpp/server/ServerClientLocal.cpp | 12 ++++++++++++ .../src/main/native/cpp/server/ServerClientLocal.h | 3 +-- ntcore/src/main/native/cpp/server/ServerImpl.cpp | 14 +++----------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp b/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp index 36c99d9ae1d..e99ca4f9785 100644 --- a/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp +++ b/ntcore/src/main/native/cpp/server/ServerClientLocal.cpp @@ -8,6 +8,18 @@ using namespace nt::server; +ServerClientLocal::ServerClientLocal(ServerStorage& storage, int id, + wpi::Logger& logger) + : ServerClient4Base{"", "", true, [](uint32_t) {}, storage, id, logger} { + // create local client meta topics + m_metaPub = storage.CreateMetaTopic("$serverpub"); + m_metaSub = storage.CreateMetaTopic("$serversub"); + + // update meta topics + UpdateMetaClientPub(); + UpdateMetaClientSub(); +} + void ServerClientLocal::SendValue(ServerTopic* topic, const Value& value, net::ValueSendMode mode) { if (m_local) { diff --git a/ntcore/src/main/native/cpp/server/ServerClientLocal.h b/ntcore/src/main/native/cpp/server/ServerClientLocal.h index 4ddd73365f0..16d0efafa25 100644 --- a/ntcore/src/main/native/cpp/server/ServerClientLocal.h +++ b/ntcore/src/main/native/cpp/server/ServerClientLocal.h @@ -13,8 +13,7 @@ namespace nt::server { class ServerClientLocal final : public ServerClient4Base { public: - ServerClientLocal(ServerStorage& storage, int id, wpi::Logger& logger) - : ServerClient4Base{"", "", true, [](uint32_t) {}, storage, id, logger} {} + ServerClientLocal(ServerStorage& storage, int id, wpi::Logger& logger); bool ProcessIncomingText(std::string_view data) final { return false; } bool ProcessIncomingBinary(std::span data) final { diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.cpp b/ntcore/src/main/native/cpp/server/ServerImpl.cpp index 8265ac71fda..3c92488c973 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.cpp +++ b/ntcore/src/main/native/cpp/server/ServerImpl.cpp @@ -33,6 +33,9 @@ ServerImpl::ServerImpl(wpi::Logger& logger) m_clients.emplace_back( std::make_unique(m_storage, 0, logger)); m_localClient = static_cast(m_clients.back().get()); + + // create server meta topics + m_metaClients = m_storage.CreateMetaTopic("$clients"); } std::pair ServerImpl::AddClient(std::string_view name, @@ -169,17 +172,6 @@ void ServerImpl::SetLocal(net::ServerMessageHandler* local, net::ClientMessageQueue* queue) { DEBUG4("SetLocal()"); m_localClient->SetLocal(local, queue); - - // create server meta topics - m_metaClients = m_storage.CreateMetaTopic("$clients"); - - // create local client meta topics - m_localClient->m_metaPub = m_storage.CreateMetaTopic("$serverpub"); - m_localClient->m_metaSub = m_storage.CreateMetaTopic("$serversub"); - - // update meta topics - m_localClient->UpdateMetaClientPub(); - m_localClient->UpdateMetaClientSub(); } bool ServerImpl::ProcessIncomingText(int clientId, std::string_view data) { From 7cda97454c57fe74146ca683bfdc61dd9d08c1a7 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Thu, 17 Oct 2024 16:27:42 -0700 Subject: [PATCH 15/26] Clean up some forward declarations --- ntcore/src/main/native/cpp/server/ServerImpl.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/ntcore/src/main/native/cpp/server/ServerImpl.h b/ntcore/src/main/native/cpp/server/ServerImpl.h index f78ee672d3b..a7ee99f1d50 100644 --- a/ntcore/src/main/native/cpp/server/ServerImpl.h +++ b/ntcore/src/main/native/cpp/server/ServerImpl.h @@ -19,9 +19,6 @@ namespace wpi { class Logger; -template -class SmallVectorImpl; -class raw_ostream; } // namespace wpi namespace nt::net { From 4941ab47aeeebc62bfa5e760f1901d5f466edd33 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Fri, 18 Oct 2024 23:08:52 -0700 Subject: [PATCH 16/26] [ntcore] Split LocalStorage implementation into separate files --- ntcore/CMakeLists.txt | 1 + ntcore/src/main/native/cpp/LocalStorage.cpp | 1540 +---------------- ntcore/src/main/native/cpp/LocalStorage.h | 548 ++---- .../main/native/cpp/local/LocalDataLogger.cpp | 29 + .../main/native/cpp/local/LocalDataLogger.h | 38 + .../native/cpp/local/LocalDataLoggerEntry.cpp | 62 + .../native/cpp/local/LocalDataLoggerEntry.h | 37 + ntcore/src/main/native/cpp/local/LocalEntry.h | 31 + .../src/main/native/cpp/local/LocalListener.h | 35 + .../native/cpp/local/LocalMultiSubscriber.h | 59 + .../main/native/cpp/local/LocalPublisher.h | 36 + .../native/cpp/local/LocalStorageImpl.cpp | 1259 ++++++++++++++ .../main/native/cpp/local/LocalStorageImpl.h | 313 ++++ .../main/native/cpp/local/LocalSubscriber.h | 51 + ntcore/src/main/native/cpp/local/LocalTopic.h | 77 + .../src/main/native/cpp/local/PubSubConfig.h | 27 + 16 files changed, 2243 insertions(+), 1900 deletions(-) create mode 100644 ntcore/src/main/native/cpp/local/LocalDataLogger.cpp create mode 100644 ntcore/src/main/native/cpp/local/LocalDataLogger.h create mode 100644 ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp create mode 100644 ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h create mode 100644 ntcore/src/main/native/cpp/local/LocalEntry.h create mode 100644 ntcore/src/main/native/cpp/local/LocalListener.h create mode 100644 ntcore/src/main/native/cpp/local/LocalMultiSubscriber.h create mode 100644 ntcore/src/main/native/cpp/local/LocalPublisher.h create mode 100644 ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp create mode 100644 ntcore/src/main/native/cpp/local/LocalStorageImpl.h create mode 100644 ntcore/src/main/native/cpp/local/LocalSubscriber.h create mode 100644 ntcore/src/main/native/cpp/local/LocalTopic.h create mode 100644 ntcore/src/main/native/cpp/local/PubSubConfig.h diff --git a/ntcore/CMakeLists.txt b/ntcore/CMakeLists.txt index c8fdca2262b..f108a9133d2 100644 --- a/ntcore/CMakeLists.txt +++ b/ntcore/CMakeLists.txt @@ -7,6 +7,7 @@ file( GLOB ntcore_native_src src/main/native/cpp/*.cpp src/generated/main/native/cpp/*.cpp + src/main/native/cpp/local/*.cpp src/main/native/cpp/net/*.cpp src/main/native/cpp/net3/*.cpp src/main/native/cpp/networktables/*.cpp diff --git a/ntcore/src/main/native/cpp/LocalStorage.cpp b/ntcore/src/main/native/cpp/LocalStorage.cpp index 494e1453090..c8e35a229cb 100644 --- a/ntcore/src/main/native/cpp/LocalStorage.cpp +++ b/ntcore/src/main/native/cpp/LocalStorage.cpp @@ -4,1214 +4,16 @@ #include "LocalStorage.h" -#include -#include -#include -#include #include -#include -#include -#include -#include -#include - -#include "IListenerStorage.h" -#include "Log.h" -#include "Types_internal.h" -#include "Value_internal.h" -#include "net/MessageHandler.h" -#include "networktables/NetworkTableValue.h" - using namespace nt; -// maximum number of local publishers / subscribers to any given topic -static constexpr size_t kMaxPublishers = 512; -static constexpr size_t kMaxSubscribers = 512; -static constexpr size_t kMaxMultiSubscribers = 512; -static constexpr size_t kMaxListeners = 512; - -static constexpr bool PrefixMatch(std::string_view name, - std::string_view prefix, bool special) { - return (!special || !prefix.empty()) && wpi::starts_with(name, prefix); -} - -std::string LocalStorage::DataLoggerEntry::MakeMetadata( - std::string_view properties) { - return fmt::format("{{\"properties\":{},\"source\":\"NT\"}}", properties); -} - -bool LocalStorage::MultiSubscriberData::Matches(std::string_view name, - bool special) { - for (auto&& prefix : prefixes) { - if (PrefixMatch(name, prefix, special)) { - return true; - } - } - return false; -} - -int LocalStorage::DataLoggerData::Start(TopicData* topic, int64_t time) { - std::string_view typeStr = topic->typeStr; - // NT and DataLog use different standard representations for int and int[] - if (typeStr == "int") { - typeStr = "int64"; - } else if (typeStr == "int[]") { - typeStr = "int64[]"; - } - return log.Start( - fmt::format( - "{}{}", logPrefix, - wpi::remove_prefix(topic->name, prefix).value_or(topic->name)), - typeStr, DataLoggerEntry::MakeMetadata(topic->propertiesStr), time); -} - -void LocalStorage::DataLoggerEntry::Append(const Value& v) { - auto time = v.time(); - switch (v.type()) { - case NT_BOOLEAN: - log->AppendBoolean(entry, v.GetBoolean(), time); - break; - case NT_INTEGER: - log->AppendInteger(entry, v.GetInteger(), time); - break; - case NT_FLOAT: - log->AppendFloat(entry, v.GetFloat(), time); - break; - case NT_DOUBLE: - log->AppendDouble(entry, v.GetDouble(), time); - break; - case NT_STRING: - log->AppendString(entry, v.GetString(), time); - break; - case NT_RAW: { - auto val = v.GetRaw(); - log->AppendRaw(entry, - {reinterpret_cast(val.data()), val.size()}, - time); - break; - } - case NT_BOOLEAN_ARRAY: - log->AppendBooleanArray(entry, v.GetBooleanArray(), time); - break; - case NT_INTEGER_ARRAY: - log->AppendIntegerArray(entry, v.GetIntegerArray(), time); - break; - case NT_FLOAT_ARRAY: - log->AppendFloatArray(entry, v.GetFloatArray(), time); - break; - case NT_DOUBLE_ARRAY: - log->AppendDoubleArray(entry, v.GetDoubleArray(), time); - break; - case NT_STRING_ARRAY: - log->AppendStringArray(entry, v.GetStringArray(), time); - break; - default: - break; - } -} - -TopicInfo LocalStorage::TopicData::GetTopicInfo() const { - TopicInfo info; - info.topic = handle; - info.name = name; - info.type = type; - info.type_str = typeStr; - info.properties = propertiesStr; - return info; -} - -void LocalStorage::Impl::NotifyTopic(TopicData* topic, - unsigned int eventFlags) { - DEBUG4("NotifyTopic({}, {})", topic->name, eventFlags); - auto topicInfo = topic->GetTopicInfo(); - if (!topic->listeners.empty()) { - m_listenerStorage.Notify(topic->listeners, eventFlags, topicInfo); - } - - wpi::SmallVector listeners; - for (auto listener : m_topicPrefixListeners) { - if (listener->multiSubscriber && - listener->multiSubscriber->Matches(topic->name, topic->special)) { - listeners.emplace_back(listener->handle); - } - } - if (!listeners.empty()) { - m_listenerStorage.Notify(listeners, eventFlags, topicInfo); - } - - if ((eventFlags & (NT_EVENT_PUBLISH | NT_EVENT_UNPUBLISH)) != 0) { - if (!m_dataloggers.empty()) { - auto now = Now(); - for (auto&& datalogger : m_dataloggers) { - if (PrefixMatch(topic->name, datalogger->prefix, topic->special)) { - auto it = std::find_if(topic->datalogs.begin(), topic->datalogs.end(), - [&](const auto& elem) { - return elem.logger == datalogger->handle; - }); - if ((eventFlags & NT_EVENT_PUBLISH) != 0 && - it == topic->datalogs.end()) { - topic->datalogs.emplace_back(datalogger->log, - datalogger->Start(topic, now), - datalogger->handle); - topic->datalogType = topic->type; - } else if ((eventFlags & NT_EVENT_UNPUBLISH) != 0 && - it != topic->datalogs.end()) { - it->log->Finish(it->entry, now); - topic->datalogType = NT_UNASSIGNED; - topic->datalogs.erase(it); - } - } - } - } - } else if ((eventFlags & NT_EVENT_PROPERTIES) != 0) { - if (!topic->datalogs.empty()) { - auto metadata = DataLoggerEntry::MakeMetadata(topic->propertiesStr); - for (auto&& datalog : topic->datalogs) { - datalog.log->SetMetadata(datalog.entry, metadata); - } - } - } -} - -void LocalStorage::Impl::CheckReset(TopicData* topic) { - if (topic->Exists()) { - return; - } - topic->lastValue = {}; - topic->lastValueNetwork = {}; - topic->lastValueFromNetwork = false; - topic->type = NT_UNASSIGNED; - topic->typeStr.clear(); - topic->flags = 0; - topic->properties = wpi::json::object(); - topic->propertiesStr = "{}"; -} - -bool LocalStorage::Impl::SetValue(TopicData* topic, const Value& value, - unsigned int eventFlags, - bool suppressIfDuplicate, - const PublisherData* publisher) { - const bool isDuplicate = topic->IsCached() && topic->lastValue == value; - DEBUG4("SetValue({}, {}, {}, {})", topic->name, value.time(), eventFlags, - isDuplicate); - if (topic->type != NT_UNASSIGNED && topic->type != value.type()) { - return false; - } - // Make sure value isn't older than last value - if (!topic->lastValue || topic->lastValue.time() == 0 || - value.time() >= topic->lastValue.time()) { - // TODO: notify option even if older value - if (!(suppressIfDuplicate && isDuplicate)) { - topic->type = value.type(); - if (topic->IsCached()) { - topic->lastValue = value; - topic->lastValueFromNetwork = false; - } - NotifyValue(topic, value, eventFlags, isDuplicate, publisher); - if (topic->datalogType == value.type()) { - for (auto&& datalog : topic->datalogs) { - datalog.Append(value); - } - } - } - } - - return true; -} - -void LocalStorage::Impl::NotifyValue(TopicData* topic, const Value& value, - unsigned int eventFlags, bool isDuplicate, - const PublisherData* publisher) { - bool isNetwork = (eventFlags & NT_EVENT_VALUE_REMOTE) != 0; - for (auto&& subscriber : topic->localSubscribers) { - if (subscriber->active && - (subscriber->config.keepDuplicates || !isDuplicate) && - ((isNetwork && !subscriber->config.disableRemote) || - (!isNetwork && !subscriber->config.disableLocal)) && - (!publisher || (publisher && (subscriber->config.excludePublisher != - publisher->handle)))) { - subscriber->pollStorage.emplace_back(value); - subscriber->handle.Set(); - if (!subscriber->valueListeners.empty()) { - m_listenerStorage.Notify(subscriber->valueListeners, eventFlags, - topic->handle, 0, value); - } - } - } - - for (auto&& subscriber : topic->multiSubscribers) { - if (subscriber->options.keepDuplicates || !isDuplicate) { - subscriber->handle.Set(); - if (!subscriber->valueListeners.empty()) { - m_listenerStorage.Notify(subscriber->valueListeners, eventFlags, - topic->handle, 0, value); - } - } - } -} - -void LocalStorage::Impl::SetFlags(TopicData* topic, unsigned int flags) { - wpi::json update = wpi::json::object(); - if ((flags & NT_PERSISTENT) != 0) { - topic->properties["persistent"] = true; - update["persistent"] = true; - } else { - topic->properties.erase("persistent"); - update["persistent"] = wpi::json(); - } - if ((flags & NT_RETAINED) != 0) { - topic->properties["retained"] = true; - update["retained"] = true; - } else { - topic->properties.erase("retained"); - update["retained"] = wpi::json(); - } - if ((flags & NT_UNCACHED) != 0) { - topic->properties["cached"] = false; - update["cached"] = false; - } else { - topic->properties.erase("cached"); - update["cached"] = wpi::json(); - } - if ((flags & NT_UNCACHED) != 0) { - topic->lastValue = {}; - topic->lastValueNetwork = {}; - topic->lastValueFromNetwork = false; - } - if ((flags & NT_UNCACHED) != 0 && (flags & NT_PERSISTENT) != 0) { - WARN("topic {}: disabling cached property disables persistent storage", - topic->name); - } - topic->flags = flags; - if (!update.empty()) { - PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); - } -} - -void LocalStorage::Impl::SetPersistent(TopicData* topic, bool value) { - wpi::json update = wpi::json::object(); - if (value) { - topic->flags |= NT_PERSISTENT; - topic->properties["persistent"] = true; - update["persistent"] = true; - } else { - topic->flags &= ~NT_PERSISTENT; - topic->properties.erase("persistent"); - update["persistent"] = wpi::json(); - } - PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); -} - -void LocalStorage::Impl::SetRetained(TopicData* topic, bool value) { - wpi::json update = wpi::json::object(); - if (value) { - topic->flags |= NT_RETAINED; - topic->properties["retained"] = true; - update["retained"] = true; - } else { - topic->flags &= ~NT_RETAINED; - topic->properties.erase("retained"); - update["retained"] = wpi::json(); - } - PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); -} - -void LocalStorage::Impl::SetCached(TopicData* topic, bool value) { - wpi::json update = wpi::json::object(); - if (value) { - topic->flags &= ~NT_UNCACHED; - topic->properties.erase("cached"); - update["cached"] = wpi::json(); - } else { - topic->flags |= NT_UNCACHED; - topic->properties["cached"] = false; - update["cached"] = false; - } - PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); -} - -void LocalStorage::Impl::SetProperties(TopicData* topic, - const wpi::json& update, - bool sendNetwork) { - if (!update.is_object()) { - return; - } - DEBUG4("SetProperties({},{})", topic->name, sendNetwork); - for (auto&& change : update.items()) { - if (change.value().is_null()) { - topic->properties.erase(change.key()); - } else { - topic->properties[change.key()] = change.value(); - } - } - PropertiesUpdated(topic, update, NT_EVENT_NONE, sendNetwork); -} - -void LocalStorage::Impl::PropertiesUpdated(TopicData* topic, - const wpi::json& update, - unsigned int eventFlags, - bool sendNetwork, bool updateFlags) { - DEBUG4("PropertiesUpdated({}, {}, {}, {}, {})", topic->name, update.dump(), - eventFlags, sendNetwork, updateFlags); - if (updateFlags) { - // set flags from properties - auto it = topic->properties.find("persistent"); - if (it != topic->properties.end()) { - if (auto val = it->get_ptr()) { - if (*val) { - topic->flags |= NT_PERSISTENT; - } else { - topic->flags &= ~NT_PERSISTENT; - } - } - } - it = topic->properties.find("retained"); - if (it != topic->properties.end()) { - if (auto val = it->get_ptr()) { - if (*val) { - topic->flags |= NT_RETAINED; - } else { - topic->flags &= ~NT_RETAINED; - } - } - } - it = topic->properties.find("cached"); - if (it != topic->properties.end()) { - if (auto val = it->get_ptr()) { - if (*val) { - topic->flags &= ~NT_UNCACHED; - } else { - topic->flags |= NT_UNCACHED; - } - } - } - - if ((topic->flags & NT_UNCACHED) != 0) { - topic->lastValue = {}; - topic->lastValueNetwork = {}; - topic->lastValueFromNetwork = false; - } - - if ((topic->flags & NT_UNCACHED) != 0 && - (topic->flags & NT_PERSISTENT) != 0) { - WARN("topic {}: disabling cached property disables persistent storage", - topic->name); - } - } - - topic->propertiesStr = topic->properties.dump(); - NotifyTopic(topic, eventFlags | NT_EVENT_PROPERTIES); - // check local flag so we don't echo back received properties changes - if (m_network && sendNetwork) { - m_network->ClientSetProperties(topic->name, update); - } -} - -void LocalStorage::Impl::RefreshPubSubActive(TopicData* topic, - bool warnOnSubMismatch) { - for (auto&& publisher : topic->localPublishers) { - publisher->UpdateActive(); - } - for (auto&& subscriber : topic->localSubscribers) { - subscriber->UpdateActive(); - if (warnOnSubMismatch && topic->Exists() && !subscriber->active) { - // warn on type mismatch - INFO( - "local subscribe to '{}' disabled due to type mismatch (wanted '{}', " - "published as '{}')", - topic->name, subscriber->config.typeStr, topic->typeStr); - } - } -} - -void LocalStorage::Impl::NetworkAnnounce(TopicData* topic, - std::string_view typeStr, - const wpi::json& properties, - std::optional pubuid) { - DEBUG4("LS NetworkAnnounce({}, {}, {}, {})", topic->name, typeStr, - properties.dump(), pubuid.value_or(-1)); - if (pubuid.has_value()) { - return; // ack of our publish; ignore - } - - unsigned int event = NT_EVENT_NONE; - // fresh non-local publish; the network publish always sets the type even - // if it was locally published, but output a diagnostic for this case - bool didExist = topic->Exists(); - topic->onNetwork = true; - NT_Type type = StringToType(typeStr); - if (topic->type != type || topic->typeStr != typeStr) { - if (didExist) { - INFO( - "network announce of '{}' overriding local publish (was '{}', now " - "'{}')", - topic->name, topic->typeStr, typeStr); - } - topic->type = type; - topic->typeStr = typeStr; - RefreshPubSubActive(topic, true); - } - if (!didExist) { - event |= NT_EVENT_PUBLISH; - } - - // may be properties update, but need to compare to see if it actually - // changed to determine whether to update string / send event - wpi::json update = wpi::json::object(); - // added/changed - for (auto&& prop : properties.items()) { - auto it = topic->properties.find(prop.key()); - if (it == topic->properties.end() || *it != prop.value()) { - update[prop.key()] = prop.value(); - } - } - // removed - for (auto&& prop : topic->properties.items()) { - if (properties.find(prop.key()) == properties.end()) { - update[prop.key()] = wpi::json(); - } - } - if (!update.empty()) { - topic->properties = properties; - PropertiesUpdated(topic, update, event, false); - } else if (event != NT_EVENT_NONE) { - NotifyTopic(topic, event); - } -} - -void LocalStorage::Impl::RemoveNetworkPublisher(TopicData* topic) { - DEBUG4("LS RemoveNetworkPublisher({}, {})", topic->handle.GetHandle(), - topic->name); - // this acts as an unpublish - bool didExist = topic->Exists(); - topic->onNetwork = false; - if (didExist && !topic->Exists()) { - DEBUG4("Unpublished {}", topic->name); - CheckReset(topic); - NotifyTopic(topic, NT_EVENT_UNPUBLISH); - } - - if (!topic->localPublishers.empty()) { - // some other publisher still exists; if it has a different type, refresh - // and publish it over the network - auto& nextPub = topic->localPublishers.front(); - if (nextPub->config.type != topic->type || - nextPub->config.typeStr != topic->typeStr) { - topic->type = nextPub->config.type; - topic->typeStr = nextPub->config.typeStr; - RefreshPubSubActive(topic, false); - // this may result in a duplicate publish warning on the server side, - // but send one anyway in this case just to be sure - if (nextPub->active && m_network) { - m_network->ClientPublish(Handle{nextPub->handle}.GetIndex(), - topic->name, topic->typeStr, topic->properties, - nextPub->config); - } - } - } -} - -void LocalStorage::Impl::NetworkPropertiesUpdate(TopicData* topic, - const wpi::json& update, - bool ack) { - DEBUG4("NetworkPropertiesUpdate({},{})", topic->name, ack); - if (ack) { - return; // ignore acks - } - SetProperties(topic, update, false); -} - -LocalStorage::PublisherData* LocalStorage::Impl::AddLocalPublisher( - TopicData* topic, const wpi::json& properties, const PubSubConfig& config) { - bool didExist = topic->Exists(); - auto publisher = m_publishers.Add(m_inst, topic, config); - topic->localPublishers.Add(publisher); - - if (!didExist) { - DEBUG4("AddLocalPublisher: setting {} type {} typestr {}", topic->name, - static_cast(config.type), config.typeStr); - // set the type to the published type - topic->type = config.type; - topic->typeStr = config.typeStr; - RefreshPubSubActive(topic, true); - - if (properties.is_null()) { - topic->properties = wpi::json::object(); - } else if (properties.is_object()) { - topic->properties = properties; - } else { - WARN("ignoring non-object properties when publishing '{}'", topic->name); - topic->properties = wpi::json::object(); - } - - if (topic->properties.empty()) { - NotifyTopic(topic, NT_EVENT_PUBLISH); - } else { - PropertiesUpdated(topic, topic->properties, NT_EVENT_PUBLISH, false); - } - } else { - // only need to update just this publisher - publisher->UpdateActive(); - if (!publisher->active) { - // warn on type mismatch - INFO( - "local publish to '{}' disabled due to type mismatch (wanted '{}', " - "currently '{}')", - topic->name, config.typeStr, topic->typeStr); - } - } - - if (publisher->active && m_network) { - m_network->ClientPublish(Handle{publisher->handle}.GetIndex(), topic->name, - topic->typeStr, topic->properties, config); - } - return publisher; -} - -std::unique_ptr -LocalStorage::Impl::RemoveLocalPublisher(NT_Publisher pubHandle) { - auto publisher = m_publishers.Remove(pubHandle); - if (publisher) { - auto topic = publisher->topic; - bool didExist = topic->Exists(); - topic->localPublishers.Remove(publisher.get()); - if (didExist && !topic->Exists()) { - CheckReset(topic); - NotifyTopic(topic, NT_EVENT_UNPUBLISH); - } - - if (publisher->active && m_network) { - m_network->ClientUnpublish(Handle{publisher->handle}.GetIndex()); - } - - if (publisher->active && !topic->localPublishers.empty()) { - // some other publisher still exists; if it has a different type, refresh - // and publish it over the network - auto& nextPub = topic->localPublishers.front(); - if (nextPub->config.type != topic->type || - nextPub->config.typeStr != topic->typeStr) { - topic->type = nextPub->config.type; - topic->typeStr = nextPub->config.typeStr; - RefreshPubSubActive(topic, false); - if (nextPub->active && m_network) { - m_network->ClientPublish(Handle{nextPub->handle}.GetIndex(), - topic->name, topic->typeStr, - topic->properties, nextPub->config); - } - } - } - } - return publisher; -} - -LocalStorage::SubscriberData* LocalStorage::Impl::AddLocalSubscriber( - TopicData* topic, const PubSubConfig& config) { - DEBUG4("AddLocalSubscriber({})", topic->name); - auto subscriber = m_subscribers.Add(m_inst, topic, config); - topic->localSubscribers.Add(subscriber); - // set subscriber to active if the type matches - subscriber->UpdateActive(); - if (topic->Exists() && !subscriber->active) { - // warn on type mismatch - INFO( - "local subscribe to '{}' disabled due to type mismatch (wanted '{}', " - "published as '{}')", - topic->name, config.typeStr, topic->typeStr); - } - if (m_network && !subscriber->config.hidden) { - DEBUG4("-> NetworkSubscribe({})", topic->name); - m_network->ClientSubscribe(1 + Handle{subscriber->handle}.GetIndex(), - {{topic->name}}, config); - } - - // queue current value - if (subscriber->active) { - if (!topic->lastValueFromNetwork && !config.disableLocal) { - subscriber->pollStorage.emplace_back(topic->lastValue); - subscriber->handle.Set(); - } else if (topic->lastValueFromNetwork && !config.disableRemote) { - subscriber->pollStorage.emplace_back(topic->lastValueNetwork); - subscriber->handle.Set(); - } - } - return subscriber; -} - -std::unique_ptr -LocalStorage::Impl::RemoveLocalSubscriber(NT_Subscriber subHandle) { - auto subscriber = m_subscribers.Remove(subHandle); - if (subscriber) { - auto topic = subscriber->topic; - topic->localSubscribers.Remove(subscriber.get()); - for (auto&& listener : m_listeners) { - if (listener.getSecond()->subscriber == subscriber.get()) { - listener.getSecond()->subscriber = nullptr; - } - } - if (m_network && !subscriber->config.hidden) { - m_network->ClientUnsubscribe(1 + Handle{subscriber->handle}.GetIndex()); - } - } - return subscriber; -} - -LocalStorage::EntryData* LocalStorage::Impl::AddEntry( - SubscriberData* subscriber) { - auto entry = m_entries.Add(m_inst, subscriber); - subscriber->topic->entries.Add(entry); - return entry; -} - -std::unique_ptr LocalStorage::Impl::RemoveEntry( - NT_Entry entryHandle) { - auto entry = m_entries.Remove(entryHandle); - if (entry) { - entry->topic->entries.Remove(entry.get()); - } - return entry; -} - -LocalStorage::MultiSubscriberData* LocalStorage::Impl::AddMultiSubscriber( - std::span prefixes, const PubSubOptions& options) { - DEBUG4("AddMultiSubscriber({})", fmt::join(prefixes, ",")); - auto subscriber = m_multiSubscribers.Add(m_inst, prefixes, options); - // subscribe to any already existing topics - for (auto&& topic : m_topics) { - for (auto&& prefix : prefixes) { - if (PrefixMatch(topic->name, prefix, topic->special)) { - topic->multiSubscribers.Add(subscriber); - break; - } - } - } - if (m_network && !subscriber->options.hidden) { - DEBUG4("-> NetworkSubscribe"); - m_network->ClientSubscribe(-1 - Handle{subscriber->handle}.GetIndex(), - subscriber->prefixes, subscriber->options); - } - return subscriber; -} - -std::unique_ptr -LocalStorage::Impl::RemoveMultiSubscriber(NT_MultiSubscriber subHandle) { - auto subscriber = m_multiSubscribers.Remove(subHandle); - if (subscriber) { - for (auto&& topic : m_topics) { - topic->multiSubscribers.Remove(subscriber.get()); - } - for (auto&& listener : m_listeners) { - if (listener.getSecond()->multiSubscriber == subscriber.get()) { - listener.getSecond()->multiSubscriber = nullptr; - } - } - if (m_network && !subscriber->options.hidden) { - m_network->ClientUnsubscribe(-1 - Handle{subscriber->handle}.GetIndex()); - } - } - return subscriber; -} - -void LocalStorage::Impl::AddListenerImpl(NT_Listener listenerHandle, - TopicData* topic, - unsigned int eventMask) { - if (topic->localSubscribers.size() >= kMaxSubscribers) { - ERR("reached maximum number of subscribers to '{}', ignoring listener add", - topic->name); - return; - } - // subscribe to make sure topic updates are received - PubSubConfig config; - config.topicsOnly = (eventMask & NT_EVENT_VALUE_ALL) == 0; - auto sub = AddLocalSubscriber(topic, config); - AddListenerImpl(listenerHandle, sub, eventMask, sub->handle, true); -} - -void LocalStorage::Impl::AddListenerImpl(NT_Listener listenerHandle, - SubscriberData* subscriber, - unsigned int eventMask, - NT_Handle subentryHandle, - bool subscriberOwned) { - m_listeners.try_emplace(listenerHandle, std::make_unique( - listenerHandle, subscriber, - eventMask, subscriberOwned)); - - auto topic = subscriber->topic; - - if ((eventMask & NT_EVENT_TOPIC) != 0) { - if (topic->listeners.size() >= kMaxListeners) { - ERR("reached maximum number of listeners to '{}', not adding listener", - topic->name); - return; - } - - m_listenerStorage.Activate( - listenerHandle, eventMask & (NT_EVENT_TOPIC | NT_EVENT_IMMEDIATE)); - - topic->listeners.Add(listenerHandle); - - // handle immediate publish - if ((eventMask & (NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE)) == - (NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE) && - topic->Exists()) { - m_listenerStorage.Notify({&listenerHandle, 1}, - NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE, - topic->GetTopicInfo()); - } - } - - if ((eventMask & NT_EVENT_VALUE_ALL) != 0) { - if (subscriber->valueListeners.size() >= kMaxListeners) { - ERR("reached maximum number of listeners to '{}', not adding listener", - topic->name); - return; - } - m_listenerStorage.Activate( - listenerHandle, eventMask & (NT_EVENT_VALUE_ALL | NT_EVENT_IMMEDIATE), - [subentryHandle](unsigned int mask, Event* event) { - if (auto valueData = event->GetValueEventData()) { - valueData->subentry = subentryHandle; - } - return true; - }); - - subscriber->valueListeners.Add(listenerHandle); - - // handle immediate value - if ((eventMask & NT_EVENT_VALUE_ALL) != 0 && - (eventMask & NT_EVENT_IMMEDIATE) != 0 && topic->lastValue) { - m_listenerStorage.Notify({&listenerHandle, 1}, - NT_EVENT_IMMEDIATE | NT_EVENT_VALUE_ALL, - topic->handle, subentryHandle, topic->lastValue); - } - } -} - -void LocalStorage::Impl::AddListenerImpl(NT_Listener listenerHandle, - MultiSubscriberData* subscriber, - unsigned int eventMask, - bool subscriberOwned) { - auto listener = - m_listeners - .try_emplace(listenerHandle, std::make_unique( - listenerHandle, subscriber, - eventMask, subscriberOwned)) - .first->getSecond() - .get(); - - // if we're doing anything immediate, get the list of matching topics - wpi::SmallVector topics; - if ((eventMask & NT_EVENT_IMMEDIATE) != 0 && - (eventMask & (NT_EVENT_PUBLISH | NT_EVENT_VALUE_ALL)) != 0) { - for (auto&& topic : m_topics) { - if (topic->Exists() && subscriber->Matches(topic->name, topic->special)) { - topics.emplace_back(topic.get()); - } - } - } - - if ((eventMask & NT_EVENT_TOPIC) != 0) { - if (m_topicPrefixListeners.size() >= kMaxListeners) { - ERR("reached maximum number of listeners, not adding listener"); - return; - } - - m_listenerStorage.Activate( - listenerHandle, eventMask & (NT_EVENT_TOPIC | NT_EVENT_IMMEDIATE)); - - m_topicPrefixListeners.Add(listener); - - // handle immediate publish - if ((eventMask & (NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE)) == - (NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE)) { - std::vector topicInfos; - for (auto&& topic : topics) { - topicInfos.emplace_back(topic->GetTopicInfo()); - } - if (!topicInfos.empty()) { - m_listenerStorage.Notify({&listenerHandle, 1}, - NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE, - topicInfos); - } - } - } - - if ((eventMask & NT_EVENT_VALUE_ALL) != 0) { - if (subscriber->valueListeners.size() >= kMaxListeners) { - ERR("reached maximum number of listeners, not adding listener"); - return; - } - - m_listenerStorage.Activate( - listenerHandle, eventMask & (NT_EVENT_VALUE_ALL | NT_EVENT_IMMEDIATE), - [subentryHandle = subscriber->handle.GetHandle()](unsigned int mask, - Event* event) { - if (auto valueData = event->GetValueEventData()) { - valueData->subentry = subentryHandle; - } - return true; - }); - - subscriber->valueListeners.Add(listenerHandle); - - // handle immediate value - if ((eventMask & NT_EVENT_VALUE_ALL) != 0 && - (eventMask & NT_EVENT_IMMEDIATE) != 0) { - for (auto&& topic : topics) { - if (topic->lastValue) { - m_listenerStorage.Notify( - {&listenerHandle, 1}, NT_EVENT_VALUE_ALL | NT_EVENT_IMMEDIATE, - topic->handle, subscriber->handle, topic->lastValue); - } - } - } - } -} - -LocalStorage::TopicData* LocalStorage::Impl::GetOrCreateTopic( - std::string_view name) { - auto& topic = m_nameTopics[name]; - // create if it does not already exist - if (!topic) { - topic = m_topics.Add(m_inst, name); - // attach multi-subscribers - for (auto&& sub : m_multiSubscribers) { - if (sub->Matches(name, topic->special)) { - topic->multiSubscribers.Add(sub.get()); - } - } - } - return topic; -} - -LocalStorage::TopicData* LocalStorage::Impl::GetTopic(NT_Handle handle) { - switch (Handle{handle}.GetType()) { - case Handle::kEntry: { - if (auto entry = m_entries.Get(handle)) { - return entry->topic; - } - break; - } - case Handle::kSubscriber: { - if (auto subscriber = m_subscribers.Get(handle)) { - return subscriber->topic; - } - break; - } - case Handle::kPublisher: { - if (auto publisher = m_publishers.Get(handle)) { - return publisher->topic; - } - break; - } - case Handle::kTopic: - return m_topics.Get(handle); - default: - break; - } - return {}; -} - -LocalStorage::SubscriberData* LocalStorage::Impl::GetSubEntry( - NT_Handle subentryHandle) { - Handle h{subentryHandle}; - if (h.IsType(Handle::kSubscriber)) { - return m_subscribers.Get(subentryHandle); - } else if (h.IsType(Handle::kEntry)) { - auto entry = m_entries.Get(subentryHandle); - return entry ? entry->subscriber : nullptr; - } else { - return nullptr; - } -} - -LocalStorage::PublisherData* LocalStorage::Impl::PublishEntry(EntryData* entry, - NT_Type type) { - if (entry->publisher) { - return entry->publisher; - } - if (entry->subscriber->config.type == NT_UNASSIGNED) { - auto typeStr = TypeToString(type); - entry->subscriber->config.type = type; - entry->subscriber->config.typeStr = typeStr; - } else if (entry->subscriber->config.type != type) { - if (!IsNumericCompatible(type, entry->subscriber->config.type)) { - // don't allow dynamically changing the type of an entry - auto typeStr = TypeToString(type); - ERR("cannot publish entry {} as type {}, previously subscribed as {}", - entry->topic->name, typeStr, entry->subscriber->config.typeStr); - return nullptr; - } - } - // create publisher - entry->publisher = AddLocalPublisher(entry->topic, wpi::json::object(), - entry->subscriber->config); - // exclude publisher if requested - if (entry->subscriber->config.excludeSelf) { - entry->subscriber->config.excludePublisher = entry->publisher->handle; - } - return entry->publisher; -} - -bool LocalStorage::Impl::PublishLocalValue(PublisherData* publisher, - const Value& value, bool force) { - if (!value) { - return false; - } - if (publisher->topic->type != NT_UNASSIGNED && - publisher->topic->type != value.type()) { - if (IsNumericCompatible(publisher->topic->type, value.type())) { - return PublishLocalValue( - publisher, ConvertNumericValue(value, publisher->topic->type)); - } - return false; - } - if (publisher->active) { - bool isNetworkDuplicate, suppressDuplicates; - if (force || publisher->config.keepDuplicates) { - suppressDuplicates = false; - isNetworkDuplicate = false; - } else { - suppressDuplicates = true; - isNetworkDuplicate = publisher->topic->IsCached() && - (publisher->topic->lastValueNetwork == value); - } - if (!isNetworkDuplicate && m_network) { - if (publisher->topic->IsCached()) { - publisher->topic->lastValueNetwork = value; - } - m_network->ClientSetValue(Handle{publisher->handle}.GetIndex(), value); - } - return SetValue(publisher->topic, value, NT_EVENT_VALUE_LOCAL, - suppressDuplicates, publisher); - } else { - return false; - } -} - -bool LocalStorage::Impl::SetEntryValue(NT_Handle pubentryHandle, - const Value& value) { - if (!value) { - return false; - } - auto publisher = m_publishers.Get(pubentryHandle); - if (!publisher) { - if (auto entry = m_entries.Get(pubentryHandle)) { - publisher = PublishEntry(entry, value.type()); - } - if (!publisher) { - return false; - } - } - return PublishLocalValue(publisher, value); -} - -bool LocalStorage::Impl::SetDefaultEntryValue(NT_Handle pubsubentryHandle, - const Value& value) { - DEBUG4("SetDefaultEntryValue({}, {})", pubsubentryHandle, - static_cast(value.type())); - if (!value) { - return false; - } - if (auto topic = GetTopic(pubsubentryHandle)) { - if (!topic->IsCached()) { - WARN("ignoring default value on non-cached topic '{}'", topic->name); - return false; - } - if (!topic->lastValue && - (topic->type == NT_UNASSIGNED || topic->type == value.type() || - IsNumericCompatible(topic->type, value.type()))) { - // publish if we haven't yet - auto publisher = m_publishers.Get(pubsubentryHandle); - if (!publisher) { - if (auto entry = m_entries.Get(pubsubentryHandle)) { - publisher = PublishEntry(entry, value.type()); - } - } - - // force value timestamps to 0 - if (topic->type == NT_UNASSIGNED) { - topic->type = value.type(); - } - Value newValue; - if (topic->type == value.type()) { - newValue = value; - } else if (IsNumericCompatible(topic->type, value.type())) { - newValue = ConvertNumericValue(value, topic->type); - } else { - return true; - } - newValue.SetTime(0); - newValue.SetServerTime(0); - if (publisher) { - PublishLocalValue(publisher, newValue, true); - } else { - topic->lastValue = newValue; - } - return true; - } - } - return false; -} - -void LocalStorage::Impl::RemoveSubEntry(NT_Handle subentryHandle) { - Handle h{subentryHandle}; - if (h.IsType(Handle::kSubscriber)) { - RemoveLocalSubscriber(subentryHandle); - } else if (h.IsType(Handle::kMultiSubscriber)) { - RemoveMultiSubscriber(subentryHandle); - } else if (h.IsType(Handle::kEntry)) { - if (auto entry = RemoveEntry(subentryHandle)) { - RemoveLocalSubscriber(entry->subscriber->handle); - if (entry->publisher) { - RemoveLocalPublisher(entry->publisher->handle); - } - } - } -} - -LocalStorage::Impl::Impl(int inst, IListenerStorage& listenerStorage, - wpi::Logger& logger) - : m_inst{inst}, m_listenerStorage{listenerStorage}, m_logger{logger} {} - -LocalStorage::~LocalStorage() = default; - -int LocalStorage::ServerAnnounce(std::string_view name, int id, - std::string_view typeStr, - const wpi::json& properties, - std::optional pubuid) { - std::scoped_lock lock{m_mutex}; - auto topic = m_impl.GetOrCreateTopic(name); - m_impl.NetworkAnnounce(topic, typeStr, properties, pubuid); - return Handle{topic->handle}.GetIndex(); -} - -void LocalStorage::ServerUnannounce(std::string_view name, int id) { - std::scoped_lock lock{m_mutex}; - auto topic = m_impl.GetOrCreateTopic(name); - m_impl.RemoveNetworkPublisher(topic); -} - -void LocalStorage::ServerPropertiesUpdate(std::string_view name, - const wpi::json& update, bool ack) { - std::scoped_lock lock{m_mutex}; - auto it = m_impl.m_nameTopics.find(name); - if (it != m_impl.m_nameTopics.end()) { - m_impl.NetworkPropertiesUpdate(it->second, update, ack); - } -} - -void LocalStorage::ServerSetValue(int topicId, const Value& value) { - std::scoped_lock lock{m_mutex}; - if (auto topic = - m_impl.m_topics.Get(Handle{m_impl.m_inst, topicId, Handle::kTopic})) { - if (m_impl.SetValue(topic, value, NT_EVENT_VALUE_REMOTE, false, nullptr)) { - if (topic->IsCached()) { - topic->lastValueNetwork = value; - topic->lastValueFromNetwork = true; - } - } - } -} - -void LocalStorage::StartNetwork(net::ClientMessageHandler* network) { - std::scoped_lock lock{m_mutex}; - m_impl.StartNetwork(network); -} - -void LocalStorage::Impl::StartNetwork(net::ClientMessageHandler* network) { - DEBUG4("StartNetwork()"); - m_network = network; - // publish all active publishers to the network and send last values - // only send value once per topic - for (auto&& topic : m_topics) { - PublisherData* anyPublisher = nullptr; - for (auto&& publisher : topic->localPublishers) { - if (publisher->active) { - network->ClientPublish(Handle{publisher->handle}.GetIndex(), - topic->name, topic->typeStr, topic->properties, - publisher->config); - anyPublisher = publisher; - } - } - if (anyPublisher && topic->lastValue) { - network->ClientSetValue(Handle{anyPublisher->handle}.GetIndex(), - topic->lastValue); - } - } - for (auto&& subscriber : m_subscribers) { - if (!subscriber->config.hidden) { - network->ClientSubscribe(1 + Handle{subscriber->handle}.GetIndex(), - {{subscriber->topic->name}}, subscriber->config); - } - } - for (auto&& subscriber : m_multiSubscribers) { - if (!subscriber->options.hidden) { - network->ClientSubscribe(-1 - Handle{subscriber->handle}.GetIndex(), - subscriber->prefixes, subscriber->options); - } - } -} - -void LocalStorage::ClearNetwork() { - WPI_DEBUG4(m_impl.m_logger, "ClearNetwork()"); - std::scoped_lock lock{m_mutex}; - m_impl.m_network = nullptr; - // treat as an unannounce all from the network side - for (auto&& topic : m_impl.m_topics) { - m_impl.RemoveNetworkPublisher(topic.get()); - } -} - -template -static void ForEachTopic(T& topics, std::string_view prefix, unsigned int types, - F func) { - for (auto&& topic : topics) { - if (!topic->Exists()) { - continue; - } - if (!wpi::starts_with(topic->name, prefix)) { - continue; - } - if (types != 0 && (types & topic->type) == 0) { - continue; - } - func(*topic); - } -} - -template -static void ForEachTopic(T& topics, std::string_view prefix, - std::span types, F func) { - for (auto&& topic : topics) { - if (!topic->Exists()) { - continue; - } - if (!wpi::starts_with(topic->name, prefix)) { - continue; - } - if (!types.empty()) { - bool match = false; - for (auto&& type : types) { - if (topic->typeStr == type) { - match = true; - break; - } - } - if (!match) { - continue; - } - } - func(*topic); - } -} - std::vector LocalStorage::GetTopics(std::string_view prefix, unsigned int types) { std::scoped_lock lock(m_mutex); std::vector rv; - ForEachTopic(m_impl.m_topics, prefix, types, - [&](TopicData& topic) { rv.push_back(topic.handle); }); + m_impl.ForEachTopic(prefix, types, + [&](auto& topic) { rv.push_back(topic.handle); }); return rv; } @@ -1219,8 +21,8 @@ std::vector LocalStorage::GetTopics( std::string_view prefix, std::span types) { std::scoped_lock lock(m_mutex); std::vector rv; - ForEachTopic(m_impl.m_topics, prefix, types, - [&](TopicData& topic) { rv.push_back(topic.handle); }); + m_impl.ForEachTopic(prefix, types, + [&](auto& topic) { rv.push_back(topic.handle); }); return rv; } @@ -1228,7 +30,7 @@ std::vector LocalStorage::GetTopicInfo(std::string_view prefix, unsigned int types) { std::scoped_lock lock(m_mutex); std::vector rv; - ForEachTopic(m_impl.m_topics, prefix, types, [&](TopicData& topic) { + m_impl.ForEachTopic(prefix, types, [&](auto& topic) { rv.emplace_back(topic.GetTopicInfo()); }); return rv; @@ -1238,166 +40,12 @@ std::vector LocalStorage::GetTopicInfo( std::string_view prefix, std::span types) { std::scoped_lock lock(m_mutex); std::vector rv; - ForEachTopic(m_impl.m_topics, prefix, types, [&](TopicData& topic) { + m_impl.ForEachTopic(prefix, types, [&](auto& topic) { rv.emplace_back(topic.GetTopicInfo()); }); return rv; } -void LocalStorage::SetTopicProperty(NT_Topic topicHandle, std::string_view name, - const wpi::json& value) { - std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { - if (value.is_null()) { - topic->properties.erase(name); - } else { - topic->properties[name] = value; - } - wpi::json update = wpi::json::object(); - update[name] = value; - m_impl.PropertiesUpdated(topic, update, NT_EVENT_NONE, true); - } -} - -void LocalStorage::DeleteTopicProperty(NT_Topic topicHandle, - std::string_view name) { - std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { - topic->properties.erase(name); - wpi::json update = wpi::json::object(); - update[name] = wpi::json(); - m_impl.PropertiesUpdated(topic, update, NT_EVENT_NONE, true); - } -} - -bool LocalStorage::SetTopicProperties(NT_Topic topicHandle, - const wpi::json& update) { - if (!update.is_object()) { - return false; - } - std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { - m_impl.SetProperties(topic, update, true); - return true; - } else { - return {}; - } -} - -NT_Subscriber LocalStorage::Subscribe(NT_Topic topicHandle, NT_Type type, - std::string_view typeStr, - const PubSubOptions& options) { - std::scoped_lock lock{m_mutex}; - - // Get the topic - auto* topic = m_impl.m_topics.Get(topicHandle); - if (!topic) { - return 0; - } - - if (topic->localSubscribers.size() >= kMaxSubscribers) { - WPI_ERROR(m_impl.m_logger, - "reached maximum number of subscribers to '{}', not subscribing", - topic->name); - return 0; - } - - // Create subscriber - return m_impl.AddLocalSubscriber(topic, PubSubConfig{type, typeStr, options}) - ->handle; -} - -NT_MultiSubscriber LocalStorage::SubscribeMultiple( - std::span prefixes, const PubSubOptions& options) { - std::scoped_lock lock{m_mutex}; - - if (m_impl.m_multiSubscribers.size() >= kMaxMultiSubscribers) { - WPI_ERROR(m_impl.m_logger, - "reached maximum number of multi-subscribers, not subscribing"); - return 0; - } - - return m_impl.AddMultiSubscriber(prefixes, options)->handle; -} - -NT_Publisher LocalStorage::Publish(NT_Topic topicHandle, NT_Type type, - std::string_view typeStr, - const wpi::json& properties, - const PubSubOptions& options) { - std::scoped_lock lock{m_mutex}; - - // Get the topic - auto* topic = m_impl.m_topics.Get(topicHandle); - if (!topic) { - WPI_ERROR(m_impl.m_logger, "trying to publish invalid topic handle ({})", - topicHandle); - return 0; - } - - if (type == NT_UNASSIGNED || typeStr.empty()) { - WPI_ERROR( - m_impl.m_logger, - "cannot publish '{}' with an unassigned type or empty type string", - topic->name); - return 0; - } - - if (topic->localPublishers.size() >= kMaxPublishers) { - WPI_ERROR(m_impl.m_logger, - "reached maximum number of publishers to '{}', not publishing", - topic->name); - return 0; - } - - return m_impl - .AddLocalPublisher(topic, properties, - PubSubConfig{type, typeStr, options}) - ->handle; -} - -void LocalStorage::Unpublish(NT_Handle pubentryHandle) { - std::scoped_lock lock{m_mutex}; - - if (Handle{pubentryHandle}.IsType(Handle::kPublisher)) { - m_impl.RemoveLocalPublisher(pubentryHandle); - } else if (auto entry = m_impl.m_entries.Get(pubentryHandle)) { - if (entry->publisher) { - m_impl.RemoveLocalPublisher(entry->publisher->handle); - entry->publisher = nullptr; - } - } else { - // TODO: report warning - return; - } -} - -NT_Entry LocalStorage::GetEntry(NT_Topic topicHandle, NT_Type type, - std::string_view typeStr, - const PubSubOptions& options) { - std::scoped_lock lock{m_mutex}; - - // Get the topic - auto* topic = m_impl.m_topics.Get(topicHandle); - if (!topic) { - return 0; - } - - if (topic->localSubscribers.size() >= kMaxSubscribers) { - WPI_ERROR( - m_impl.m_logger, - "reached maximum number of subscribers to '{}', not creating entry", - topic->name); - return 0; - } - - // Create subscriber - auto subscriber = - m_impl.AddLocalSubscriber(topic, PubSubConfig{type, typeStr, options}); - - // Create entry - return m_impl.AddEntry(subscriber)->handle; -} - void LocalStorage::Release(NT_Handle pubsubentryHandle) { switch (Handle{pubsubentryHandle}.GetType()) { case Handle::kEntry: @@ -1433,186 +81,18 @@ Value LocalStorage::GetEntryValue(NT_Handle subentryHandle) { return {}; } -NT_Entry LocalStorage::GetEntry(std::string_view name) { - if (name.empty()) { - return {}; - } - - std::scoped_lock lock{m_mutex}; - - // Get the topic data - auto* topic = m_impl.GetOrCreateTopic(name); - - if (topic->entry == 0) { - if (topic->localSubscribers.size() >= kMaxSubscribers) { - WPI_ERROR( - m_impl.m_logger, - "reached maximum number of subscribers to '{}', not creating entry", - topic->name); - return 0; - } - - // Create subscriber - auto* subscriber = m_impl.AddLocalSubscriber(topic, {}); - - // Create entry - topic->entry = m_impl.AddEntry(subscriber)->handle; - } - - return topic->entry; -} - -void LocalStorage::AddListener(NT_Listener listenerHandle, - std::span prefixes, - unsigned int mask) { - mask &= (NT_EVENT_TOPIC | NT_EVENT_VALUE_ALL | NT_EVENT_IMMEDIATE); - std::scoped_lock lock{m_mutex}; - if (m_impl.m_multiSubscribers.size() >= kMaxMultiSubscribers) { - WPI_ERROR( - m_impl.m_logger, - "reached maximum number of multi-subscribers, not adding listener"); - return; - } - // subscribe to make sure topic updates are received - auto sub = m_impl.AddMultiSubscriber( - prefixes, {.topicsOnly = (mask & NT_EVENT_VALUE_ALL) == 0}); - m_impl.AddListenerImpl(listenerHandle, sub, mask, true); -} - void LocalStorage::AddListener(NT_Listener listenerHandle, NT_Handle handle, unsigned int mask) { mask &= (NT_EVENT_TOPIC | NT_EVENT_VALUE_ALL | NT_EVENT_IMMEDIATE); std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(handle)) { + if (auto topic = m_impl.GetTopicByHandle(handle)) { m_impl.AddListenerImpl(listenerHandle, topic, mask); - } else if (auto sub = m_impl.m_multiSubscribers.Get(handle)) { + } else if (auto sub = m_impl.GetMultiSubscriberByHandle(handle)) { m_impl.AddListenerImpl(listenerHandle, sub, mask, false); - } else if (auto sub = m_impl.m_subscribers.Get(handle)) { + } else if (auto sub = m_impl.GetSubscriberByHandle(handle)) { m_impl.AddListenerImpl(listenerHandle, sub, mask, sub->handle, false); - } else if (auto entry = m_impl.m_entries.Get(handle)) { + } else if (auto entry = m_impl.GetEntryByHandle(handle)) { m_impl.AddListenerImpl(listenerHandle, entry->subscriber, mask, entry->handle, false); } } - -void LocalStorage::RemoveListener(NT_Listener listenerHandle, - unsigned int mask) { - std::scoped_lock lock{m_mutex}; - auto listenerIt = m_impl.m_listeners.find(listenerHandle); - if (listenerIt == m_impl.m_listeners.end()) { - return; - } - auto listener = std::move(listenerIt->getSecond()); - m_impl.m_listeners.erase(listenerIt); - if (!listener) { - return; - } - - m_impl.m_topicPrefixListeners.Remove(listener.get()); - if (listener->subscriber) { - listener->subscriber->valueListeners.Remove(listenerHandle); - listener->subscriber->topic->listeners.Remove(listenerHandle); - if (listener->subscriberOwned) { - m_impl.RemoveLocalSubscriber(listener->subscriber->handle); - } - } - if (listener->multiSubscriber) { - listener->multiSubscriber->valueListeners.Remove(listenerHandle); - if (listener->subscriberOwned) { - m_impl.RemoveMultiSubscriber(listener->multiSubscriber->handle); - } - } -} - -NT_DataLogger LocalStorage::StartDataLog(wpi::log::DataLog& log, - std::string_view prefix, - std::string_view logPrefix) { - std::scoped_lock lock{m_mutex}; - auto datalogger = - m_impl.m_dataloggers.Add(m_impl.m_inst, log, prefix, logPrefix); - - // start logging any matching topics - auto now = nt::Now(); - for (auto&& topic : m_impl.m_topics) { - if (!PrefixMatch(topic->name, prefix, topic->special) || - topic->type == NT_UNASSIGNED || topic->typeStr.empty()) { - continue; - } - topic->datalogs.emplace_back(log, datalogger->Start(topic.get(), now), - datalogger->handle); - topic->datalogType = topic->type; - - // log current value, if any - if (topic->lastValue) { - topic->datalogs.back().Append(topic->lastValue); - } - } - - return datalogger->handle; -} - -void LocalStorage::StopDataLog(NT_DataLogger logger) { - std::scoped_lock lock{m_mutex}; - if (auto datalogger = m_impl.m_dataloggers.Remove(logger)) { - // finish any active entries - auto now = Now(); - for (auto&& topic : m_impl.m_topics) { - auto it = - std::find_if(topic->datalogs.begin(), topic->datalogs.end(), - [&](const auto& elem) { return elem.logger == logger; }); - if (it != topic->datalogs.end()) { - it->log->Finish(it->entry, now); - topic->datalogs.erase(it); - } - } - } -} - -bool LocalStorage::HasSchema(std::string_view name) { - std::scoped_lock lock{m_mutex}; - wpi::SmallString<128> fullName{"/.schema/"}; - fullName += name; - auto it = m_impl.m_schemas.find(fullName); - return it != m_impl.m_schemas.end(); -} - -void LocalStorage::AddSchema(std::string_view name, std::string_view type, - std::span schema) { - std::scoped_lock lock{m_mutex}; - wpi::SmallString<128> fullName{"/.schema/"}; - fullName += name; - auto& pubHandle = m_impl.m_schemas[fullName]; - if (pubHandle != 0) { - return; - } - - auto topic = m_impl.GetOrCreateTopic(fullName); - - if (topic->localPublishers.size() >= kMaxPublishers) { - WPI_ERROR(m_impl.m_logger, - "reached maximum number of publishers to '{}', not publishing", - topic->name); - return; - } - - pubHandle = m_impl - .AddLocalPublisher(topic, {{"retained", true}}, - PubSubConfig{NT_RAW, type, {}}) - ->handle; - - m_impl.SetDefaultEntryValue(pubHandle, Value::MakeRaw(schema)); -} - -void LocalStorage::Reset() { - std::scoped_lock lock{m_mutex}; - m_impl.m_network = nullptr; - m_impl.m_topics.clear(); - m_impl.m_publishers.clear(); - m_impl.m_subscribers.clear(); - m_impl.m_entries.clear(); - m_impl.m_multiSubscribers.clear(); - m_impl.m_dataloggers.clear(); - m_impl.m_nameTopics.clear(); - m_impl.m_listeners.clear(); - m_impl.m_topicPrefixListeners.clear(); -} diff --git a/ntcore/src/main/native/cpp/LocalStorage.h b/ntcore/src/main/native/cpp/LocalStorage.h index 007e2b51a60..30c8c5b2bfb 100644 --- a/ntcore/src/main/native/cpp/LocalStorage.h +++ b/ntcore/src/main/native/cpp/LocalStorage.h @@ -6,25 +6,17 @@ #include -#include +#include #include #include #include -#include #include -#include -#include -#include +#include #include #include -#include "Handle.h" -#include "HandleMap.h" -#include "PubSubOptions.h" -#include "Types_internal.h" -#include "ValueCircularBuffer.h" -#include "VectorSet.h" +#include "local/LocalStorageImpl.h" #include "net/MessageHandler.h" #include "net/NetworkInterface.h" #include "ntcore_cpp.h" @@ -43,19 +35,46 @@ class LocalStorage final : public net::ILocalStorage { : m_impl{inst, listenerStorage, logger} {} LocalStorage(const LocalStorage&) = delete; LocalStorage& operator=(const LocalStorage&) = delete; - ~LocalStorage() final; // network interface functions int ServerAnnounce(std::string_view name, int id, std::string_view typeStr, const wpi::json& properties, - std::optional pubuid) final; - void ServerUnannounce(std::string_view name, int id) final; + std::optional pubuid) final { + std::scoped_lock lock{m_mutex}; + auto topic = m_impl.GetOrCreateTopic(name); + m_impl.NetworkAnnounce(topic, typeStr, properties, pubuid); + return Handle{topic->handle}.GetIndex(); + } + + void ServerUnannounce(std::string_view name, int id) final { + std::scoped_lock lock{m_mutex}; + m_impl.RemoveNetworkPublisher(m_impl.GetOrCreateTopic(name)); + } + void ServerPropertiesUpdate(std::string_view name, const wpi::json& update, - bool ack) final; - void ServerSetValue(int topicId, const Value& value) final; + bool ack) final { + std::scoped_lock lock{m_mutex}; + if (auto topic = m_impl.GetTopicByName(name)) { + m_impl.NetworkPropertiesUpdate(topic, update, ack); + } + } + + void ServerSetValue(int topicId, const Value& value) final { + std::scoped_lock lock{m_mutex}; + if (auto topic = m_impl.GetTopicById(topicId)) { + m_impl.ServerSetValue(topic, value); + } + } - void StartNetwork(net::ClientMessageHandler* network) final; - void ClearNetwork() final; + void StartNetwork(net::ClientMessageHandler* network) final { + std::scoped_lock lock{m_mutex}; + m_impl.StartNetwork(network); + } + + void ClearNetwork() final { + std::scoped_lock lock{m_mutex}; + m_impl.ClearNetwork(); + } // User functions. These are the actual implementations of the corresponding // user API functions in ntcore_cpp. @@ -79,7 +98,7 @@ class LocalStorage final : public net::ILocalStorage { std::string GetTopicName(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { return topic->name; } else { return {}; @@ -88,7 +107,7 @@ class LocalStorage final : public net::ILocalStorage { NT_Type GetTopicType(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { return topic->type; } else { return {}; @@ -97,7 +116,7 @@ class LocalStorage final : public net::ILocalStorage { std::string GetTopicTypeString(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { return topic->typeStr; } else { return {}; @@ -106,14 +125,14 @@ class LocalStorage final : public net::ILocalStorage { void SetTopicPersistent(NT_Topic topicHandle, bool value) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { m_impl.SetPersistent(topic, value); } } bool GetTopicPersistent(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { return (topic->flags & NT_PERSISTENT) != 0; } else { return false; @@ -122,14 +141,14 @@ class LocalStorage final : public net::ILocalStorage { void SetTopicRetained(NT_Topic topicHandle, bool value) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { m_impl.SetRetained(topic, value); } } bool GetTopicRetained(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { return (topic->flags & NT_RETAINED) != 0; } else { return false; @@ -138,14 +157,14 @@ class LocalStorage final : public net::ILocalStorage { void SetTopicCached(NT_Topic topicHandle, bool value) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { m_impl.SetCached(topic, value); } } bool GetTopicCached(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { return (topic->flags & NT_UNCACHED) == 0; } else { return false; @@ -154,47 +173,72 @@ class LocalStorage final : public net::ILocalStorage { bool GetTopicExists(NT_Handle handle) { std::scoped_lock lock{m_mutex}; - TopicData* topic = m_impl.GetTopic(handle); + local::TopicData* topic = m_impl.GetTopic(handle); return topic && topic->Exists(); } wpi::json GetTopicProperty(NT_Topic topicHandle, std::string_view name) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { return topic->properties.value(name, wpi::json{}); } else { return {}; } } - void SetTopicProperty(NT_Topic topic, std::string_view name, - const wpi::json& value); + void SetTopicProperty(NT_Topic topicHandle, std::string_view name, + const wpi::json& value) { + std::scoped_lock lock{m_mutex}; + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { + m_impl.SetProperty(topic, name, value); + } + } - void DeleteTopicProperty(NT_Topic topic, std::string_view name); + void DeleteTopicProperty(NT_Topic topicHandle, std::string_view name) { + std::scoped_lock lock{m_mutex}; + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { + m_impl.DeleteProperty(topic, name); + } + } wpi::json GetTopicProperties(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { return topic->properties; } else { return wpi::json::object(); } } - bool SetTopicProperties(NT_Topic topic, const wpi::json& update); + bool SetTopicProperties(NT_Topic topicHandle, const wpi::json& update) { + std::scoped_lock lock{m_mutex}; + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { + return m_impl.SetProperties(topic, update, true); + } else { + return {}; + } + } TopicInfo GetTopicInfo(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.m_topics.Get(topicHandle)) { + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { return topic->GetTopicInfo(); } else { return {}; } } - NT_Subscriber Subscribe(NT_Topic topic, NT_Type type, + NT_Subscriber Subscribe(NT_Topic topicHandle, NT_Type type, std::string_view typeStr, - const PubSubOptions& options); + const PubSubOptions& options) { + std::scoped_lock lock{m_mutex}; + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { + if (auto subscriber = m_impl.Subscribe(topic, type, typeStr, options)) { + return subscriber->handle; + } + } + return {}; + } void Unsubscribe(NT_Subscriber subHandle) { std::scoped_lock lock{m_mutex}; @@ -202,21 +246,59 @@ class LocalStorage final : public net::ILocalStorage { } NT_MultiSubscriber SubscribeMultiple( - std::span prefixes, const PubSubOptions& options); + std::span prefixes, + const PubSubOptions& options) { + std::scoped_lock lock{m_mutex}; + if (auto sub = m_impl.AddMultiSubscriber(prefixes, options)) { + return sub->handle; + } else { + return {}; + } + } void UnsubscribeMultiple(NT_MultiSubscriber subHandle) { std::scoped_lock lock{m_mutex}; m_impl.RemoveMultiSubscriber(subHandle); } - NT_Publisher Publish(NT_Topic topic, NT_Type type, std::string_view typeStr, - const wpi::json& properties, - const PubSubOptions& options); + NT_Publisher Publish(NT_Topic topicHandle, NT_Type type, + std::string_view typeStr, const wpi::json& properties, + const PubSubOptions& options) { + std::scoped_lock lock{m_mutex}; + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { + if (auto publisher = + m_impl.Publish(topic, type, typeStr, properties, options)) { + return publisher->handle; + } + } + return {}; + } - void Unpublish(NT_Handle pubentry); + void Unpublish(NT_Handle pubentryHandle) { + std::scoped_lock lock{m_mutex}; + if (Handle{pubentryHandle}.IsType(Handle::kPublisher)) { + m_impl.RemoveLocalPublisher(pubentryHandle); + } else if (auto entry = m_impl.GetEntryByHandle(pubentryHandle)) { + if (entry->publisher) { + m_impl.RemoveLocalPublisher(entry->publisher->handle); + entry->publisher = nullptr; + } + } else { + // TODO: report warning + return; + } + } - NT_Entry GetEntry(NT_Topic topic, NT_Type type, std::string_view typeStr, - const PubSubOptions& options); + NT_Entry GetEntry(NT_Topic topicHandle, NT_Type type, + std::string_view typeStr, const PubSubOptions& options) { + std::scoped_lock lock{m_mutex}; + if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { + if (auto entry = m_impl.GetEntry(topic, type, typeStr, options)) { + return entry->handle; + } + } + return {}; + } void ReleaseEntry(NT_Entry entryHandle) { std::scoped_lock lock{m_mutex}; @@ -298,14 +380,14 @@ class LocalStorage final : public net::ILocalStorage { void SetEntryFlags(NT_Entry entryHandle, unsigned int flags) { std::scoped_lock lock{m_mutex}; - if (auto entry = m_impl.m_entries.Get(entryHandle)) { + if (auto entry = m_impl.GetEntryByHandle(entryHandle)) { m_impl.SetFlags(entry->subscriber->topic, flags); } } unsigned int GetEntryFlags(NT_Entry entryHandle) { std::scoped_lock lock{m_mutex}; - if (auto entry = m_impl.m_entries.Get(entryHandle)) { + if (auto entry = m_impl.GetEntryByHandle(entryHandle)) { return entry->subscriber->topic->flags; } else { return 0; @@ -313,7 +395,14 @@ class LocalStorage final : public net::ILocalStorage { } // Index-only - NT_Entry GetEntry(std::string_view name); + NT_Entry GetEntry(std::string_view name) { + std::scoped_lock lock{m_mutex}; + if (auto topic = m_impl.GetEntry(name)) { + return topic->entry; + } else { + return {}; + } + } std::string GetEntryName(NT_Entry subentryHandle) { std::scoped_lock lock{m_mutex}; @@ -346,346 +435,65 @@ class LocalStorage final : public net::ILocalStorage { // Listener functions // - void AddListener(NT_Listener listener, + void AddListener(NT_Listener listenerHandle, std::span prefixes, - unsigned int mask); + unsigned int mask) { + mask &= (NT_EVENT_TOPIC | NT_EVENT_VALUE_ALL | NT_EVENT_IMMEDIATE); + std::scoped_lock lock{m_mutex}; + // subscribe to make sure topic updates are received + if (auto sub = m_impl.AddMultiSubscriber( + prefixes, {.topicsOnly = (mask & NT_EVENT_VALUE_ALL) == 0})) { + m_impl.AddListenerImpl(listenerHandle, sub, mask, true); + } + } + void AddListener(NT_Listener listener, NT_Handle handle, unsigned int mask); - void RemoveListener(NT_Listener listener, unsigned int mask); + void RemoveListener(NT_Listener listener, unsigned int mask) { + std::scoped_lock lock{m_mutex}; + m_impl.RemoveListener(listener, mask); + } // // Data log functions // NT_DataLogger StartDataLog(wpi::log::DataLog& log, std::string_view prefix, - std::string_view logPrefix); - void StopDataLog(NT_DataLogger logger); + std::string_view logPrefix) { + std::scoped_lock lock{m_mutex}; + if (auto dl = m_impl.StartDataLog(log, prefix, logPrefix)) { + return dl->handle; + } else { + return {}; + } + } + + void StopDataLog(NT_DataLogger logger) { + std::scoped_lock lock{m_mutex}; + m_impl.StopDataLog(logger); + } // // Schema functions // - bool HasSchema(std::string_view name); - void AddSchema(std::string_view name, std::string_view type, - std::span schema); - - void Reset(); - - private: - static constexpr bool IsSpecial(std::string_view name) { - return name.empty() ? false : name.front() == '$'; + bool HasSchema(std::string_view name) { + std::scoped_lock lock{m_mutex}; + return m_impl.HasSchema(name); } - struct EntryData; - struct PublisherData; - struct SubscriberData; - struct MultiSubscriberData; - - struct DataLoggerEntry { - DataLoggerEntry(wpi::log::DataLog& log, int entry, NT_DataLogger logger) - : log{&log}, entry{entry}, logger{logger} {} - - static std::string MakeMetadata(std::string_view properties); - - void Append(const Value& v); - - wpi::log::DataLog* log; - int entry; - NT_DataLogger logger; - }; - - struct TopicData { - static constexpr auto kType = Handle::kTopic; - - TopicData(NT_Topic handle, std::string_view name) - : handle{handle}, name{name}, special{IsSpecial(name)} {} - - bool Exists() const { return onNetwork || !localPublishers.empty(); } - - bool IsCached() const { return (flags & NT_UNCACHED) == 0; } - - TopicInfo GetTopicInfo() const; - - // invariants - wpi::SignalObject handle; - std::string name; - bool special; - - Value lastValue; // also stores timestamp - Value lastValueNetwork; - NT_Type type{NT_UNASSIGNED}; - std::string typeStr; - unsigned int flags{0}; // for NT3 APIs - std::string propertiesStr{"{}"}; // cached string for GetTopicInfo() et al - wpi::json properties = wpi::json::object(); - NT_Entry entry{0}; // cached entry for GetEntry() - - bool onNetwork{false}; // true if there are any remote publishers - bool lastValueFromNetwork{false}; - - wpi::SmallVector datalogs; - NT_Type datalogType{NT_UNASSIGNED}; - - VectorSet localPublishers; - VectorSet localSubscribers; - VectorSet multiSubscribers; - VectorSet entries; - VectorSet listeners; - }; - - struct PubSubConfig : public PubSubOptionsImpl { - PubSubConfig() = default; - PubSubConfig(NT_Type type, std::string_view typeStr, - const PubSubOptions& options) - : PubSubOptionsImpl{options}, type{type}, typeStr{typeStr} { - prefixMatch = false; - } - - NT_Type type{NT_UNASSIGNED}; - std::string typeStr; - }; - - struct PublisherData { - static constexpr auto kType = Handle::kPublisher; - - PublisherData(NT_Publisher handle, TopicData* topic, PubSubConfig config) - : handle{handle}, topic{topic}, config{std::move(config)} {} - - void UpdateActive() { - active = config.type == topic->type && config.typeStr == topic->typeStr; - } - - // invariants - wpi::SignalObject handle; - TopicData* topic; - PubSubConfig config; - - // whether or not the publisher should actually publish values - bool active{false}; - }; - - struct SubscriberData { - static constexpr auto kType = Handle::kSubscriber; - - SubscriberData(NT_Subscriber handle, TopicData* topic, PubSubConfig config) - : handle{handle}, - topic{topic}, - config{std::move(config)}, - pollStorage{config.pollStorage} {} - - void UpdateActive() { - // for subscribers, unassigned is a wildcard - // also allow numerically compatible subscribers - active = - config.type == NT_UNASSIGNED || - (config.type == topic->type && config.typeStr == topic->typeStr) || - IsNumericCompatible(config.type, topic->type); - } - - // invariants - wpi::SignalObject handle; - TopicData* topic; - PubSubConfig config; - - // whether or not the subscriber should actually receive values - bool active{false}; - - // polling storage - ValueCircularBuffer pollStorage; - - // value listeners - VectorSet valueListeners; - }; - - struct EntryData { - static constexpr auto kType = Handle::kEntry; - - EntryData(NT_Entry handle, SubscriberData* subscriber) - : handle{handle}, topic{subscriber->topic}, subscriber{subscriber} {} - - // invariants - wpi::SignalObject handle; - TopicData* topic; - SubscriberData* subscriber; - - // the publisher (created on demand) - PublisherData* publisher{nullptr}; - }; - - struct MultiSubscriberData { - static constexpr auto kType = Handle::kMultiSubscriber; - - MultiSubscriberData(NT_MultiSubscriber handle, - std::span prefixes, - const PubSubOptionsImpl& options) - : handle{handle}, options{options} { - this->options.prefixMatch = true; - this->prefixes.reserve(prefixes.size()); - for (auto&& prefix : prefixes) { - this->prefixes.emplace_back(prefix); - } - } - - bool Matches(std::string_view name, bool special); - - // invariants - wpi::SignalObject handle; - std::vector prefixes; - PubSubOptionsImpl options; - - // value listeners - VectorSet valueListeners; - }; - - struct ListenerData { - ListenerData(NT_Listener handle, SubscriberData* subscriber, - unsigned int eventMask, bool subscriberOwned) - : handle{handle}, - eventMask{eventMask}, - subscriber{subscriber}, - subscriberOwned{subscriberOwned} {} - ListenerData(NT_Listener handle, MultiSubscriberData* subscriber, - unsigned int eventMask, bool subscriberOwned) - : handle{handle}, - eventMask{eventMask}, - multiSubscriber{subscriber}, - subscriberOwned{subscriberOwned} {} - - NT_Listener handle; - unsigned int eventMask; - SubscriberData* subscriber{nullptr}; - MultiSubscriberData* multiSubscriber{nullptr}; - bool subscriberOwned; - }; - - struct DataLoggerData { - static constexpr auto kType = Handle::kDataLogger; - - DataLoggerData(NT_DataLogger handle, wpi::log::DataLog& log, - std::string_view prefix, std::string_view logPrefix) - : handle{handle}, log{log}, prefix{prefix}, logPrefix{logPrefix} {} - - int Start(TopicData* topic, int64_t time); - - NT_DataLogger handle; - wpi::log::DataLog& log; - std::string prefix; - std::string logPrefix; - }; - - // inner struct to protect against accidentally deadlocking on the mutex - struct Impl { - Impl(int inst, IListenerStorage& listenerStorage, wpi::Logger& logger); - - int m_inst; - IListenerStorage& m_listenerStorage; - wpi::Logger& m_logger; - net::ClientMessageHandler* m_network{nullptr}; - - // handle mappings - HandleMap m_topics; - HandleMap m_publishers; - HandleMap m_subscribers; - HandleMap m_entries; - HandleMap m_multiSubscribers; - HandleMap m_dataloggers; - - // name mappings - wpi::StringMap m_nameTopics; - - // listeners - wpi::DenseMap> m_listeners; - - // string-based listeners - VectorSet m_topicPrefixListeners; - - // schema publishers - wpi::StringMap m_schemas; - - // topic functions - void NotifyTopic(TopicData* topic, unsigned int eventFlags); - - void CheckReset(TopicData* topic); - - bool SetValue(TopicData* topic, const Value& value, unsigned int eventFlags, - bool suppressIfDuplicate, const PublisherData* publisher); - void NotifyValue(TopicData* topic, const Value& value, - unsigned int eventFlags, bool isDuplicate, - const PublisherData* publisher); - - void SetFlags(TopicData* topic, unsigned int flags); - void SetPersistent(TopicData* topic, bool value); - void SetRetained(TopicData* topic, bool value); - void SetCached(TopicData* topic, bool value); - void SetProperties(TopicData* topic, const wpi::json& update, - bool sendNetwork); - void PropertiesUpdated(TopicData* topic, const wpi::json& update, - unsigned int eventFlags, bool sendNetwork, - bool updateFlags = true); - - void RefreshPubSubActive(TopicData* topic, bool warnOnSubMismatch); - - void NetworkAnnounce(TopicData* topic, std::string_view typeStr, - const wpi::json& properties, - std::optional pubuid); - void RemoveNetworkPublisher(TopicData* topic); - void NetworkPropertiesUpdate(TopicData* topic, const wpi::json& update, - bool ack); - void StartNetwork(net::ClientMessageHandler* network); - - PublisherData* AddLocalPublisher(TopicData* topic, - const wpi::json& properties, - const PubSubConfig& options); - std::unique_ptr RemoveLocalPublisher(NT_Publisher pubHandle); - - SubscriberData* AddLocalSubscriber(TopicData* topic, - const PubSubConfig& options); - std::unique_ptr RemoveLocalSubscriber( - NT_Subscriber subHandle); - - EntryData* AddEntry(SubscriberData* subscriber); - std::unique_ptr RemoveEntry(NT_Entry entryHandle); - - MultiSubscriberData* AddMultiSubscriber( - std::span prefixes, - const PubSubOptions& options); - std::unique_ptr RemoveMultiSubscriber( - NT_MultiSubscriber subHandle); - - void AddListenerImpl(NT_Listener listenerHandle, TopicData* topic, - unsigned int eventMask); - void AddListenerImpl(NT_Listener listenerHandle, SubscriberData* subscriber, - unsigned int eventMask, NT_Handle subentryHandle, - bool subscriberOwned); - void AddListenerImpl(NT_Listener listenerHandle, - MultiSubscriberData* subscriber, - unsigned int eventMask, bool subscriberOwned); - void AddListenerImpl(NT_Listener listenerHandle, - std::span prefixes, - unsigned int eventMask); - - TopicData* GetOrCreateTopic(std::string_view name); - TopicData* GetTopic(NT_Handle handle); - SubscriberData* GetSubEntry(NT_Handle subentryHandle); - PublisherData* PublishEntry(EntryData* entry, NT_Type type); - - Value* GetSubEntryValue(NT_Handle subentryHandle) { - if (auto subscriber = GetSubEntry(subentryHandle)) { - return &subscriber->topic->lastValue; - } else { - return nullptr; - } - } - - bool PublishLocalValue(PublisherData* publisher, const Value& value, - bool force = false); - - bool SetEntryValue(NT_Handle pubentryHandle, const Value& value); - bool SetDefaultEntryValue(NT_Handle pubsubentryHandle, const Value& value); + void AddSchema(std::string_view name, std::string_view type, + std::span schema) { + std::scoped_lock lock{m_mutex}; + m_impl.AddSchema(name, type, schema); + } - void RemoveSubEntry(NT_Handle subentryHandle); - }; + void Reset() { + std::scoped_lock lock{m_mutex}; + m_impl.Reset(); + } + private: wpi::mutex m_mutex; - Impl m_impl; + local::StorageImpl m_impl; }; } // namespace nt diff --git a/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp b/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp new file mode 100644 index 00000000000..f142ec920e0 --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp @@ -0,0 +1,29 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "LocalDataLogger.h" + +#include +#include +#include + +#include "local/LocalDataLoggerEntry.h" +#include "local/LocalTopic.h" + +using namespace nt::local; + +int DataLoggerData::Start(TopicData* topic, int64_t time) { + std::string_view typeStr = topic->typeStr; + // NT and DataLog use different standard representations for int and int[] + if (typeStr == "int") { + typeStr = "int64"; + } else if (typeStr == "int[]") { + typeStr = "int64[]"; + } + return log.Start( + fmt::format( + "{}{}", logPrefix, + wpi::remove_prefix(topic->name, prefix).value_or(topic->name)), + typeStr, DataLoggerEntry::MakeMetadata(topic->propertiesStr), time); +} diff --git a/ntcore/src/main/native/cpp/local/LocalDataLogger.h b/ntcore/src/main/native/cpp/local/LocalDataLogger.h new file mode 100644 index 00000000000..431d9346b05 --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalDataLogger.h @@ -0,0 +1,38 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include +#include + +#include "Handle.h" +#include "ntcore_c.h" + +namespace wpi::log { +class DataLog; +} // namespace wpi::log + +namespace nt::local { + +struct TopicData; + +struct DataLoggerData { + static constexpr auto kType = Handle::kDataLogger; + + DataLoggerData(NT_DataLogger handle, wpi::log::DataLog& log, + std::string_view prefix, std::string_view logPrefix) + : handle{handle}, log{log}, prefix{prefix}, logPrefix{logPrefix} {} + + int Start(TopicData* topic, int64_t time); + + NT_DataLogger handle; + wpi::log::DataLog& log; + std::string prefix; + std::string logPrefix; +}; + +} // namespace nt::local diff --git a/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp new file mode 100644 index 00000000000..de06f737f7f --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp @@ -0,0 +1,62 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "LocalDataLoggerEntry.h" + +#include +#include +#include + +#include "networktables/NetworkTableValue.h" + +using namespace nt::local; + +std::string DataLoggerEntry::MakeMetadata(std::string_view properties) { + return fmt::format("{{\"properties\":{},\"source\":\"NT\"}}", properties); +} + +void DataLoggerEntry::Append(const Value& v) { + auto time = v.time(); + switch (v.type()) { + case NT_BOOLEAN: + log->AppendBoolean(entry, v.GetBoolean(), time); + break; + case NT_INTEGER: + log->AppendInteger(entry, v.GetInteger(), time); + break; + case NT_FLOAT: + log->AppendFloat(entry, v.GetFloat(), time); + break; + case NT_DOUBLE: + log->AppendDouble(entry, v.GetDouble(), time); + break; + case NT_STRING: + log->AppendString(entry, v.GetString(), time); + break; + case NT_RAW: { + auto val = v.GetRaw(); + log->AppendRaw(entry, + {reinterpret_cast(val.data()), val.size()}, + time); + break; + } + case NT_BOOLEAN_ARRAY: + log->AppendBooleanArray(entry, v.GetBooleanArray(), time); + break; + case NT_INTEGER_ARRAY: + log->AppendIntegerArray(entry, v.GetIntegerArray(), time); + break; + case NT_FLOAT_ARRAY: + log->AppendFloatArray(entry, v.GetFloatArray(), time); + break; + case NT_DOUBLE_ARRAY: + log->AppendDoubleArray(entry, v.GetDoubleArray(), time); + break; + case NT_STRING_ARRAY: + log->AppendStringArray(entry, v.GetStringArray(), time); + break; + default: + break; + } +} diff --git a/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h new file mode 100644 index 00000000000..8b8be1a951f --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h @@ -0,0 +1,37 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +#include "ntcore_c.h" + +namespace wpi::log { +class DataLog; +} // namespace wpi::log + +namespace nt { +class Value; +} // namespace nt + +namespace nt::local { + +struct TopicData; + +struct DataLoggerEntry { + DataLoggerEntry(wpi::log::DataLog& log, int entry, NT_DataLogger logger) + : log{&log}, entry{entry}, logger{logger} {} + + static std::string MakeMetadata(std::string_view properties); + + void Append(const Value& v); + + wpi::log::DataLog* log; + int entry; + NT_DataLogger logger; +}; + +} // namespace nt::local diff --git a/ntcore/src/main/native/cpp/local/LocalEntry.h b/ntcore/src/main/native/cpp/local/LocalEntry.h new file mode 100644 index 00000000000..334e59a838e --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalEntry.h @@ -0,0 +1,31 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include "Handle.h" +#include "local/LocalSubscriber.h" + +namespace nt::local { + +struct PublisherData; + +struct EntryData { + static constexpr auto kType = Handle::kEntry; + + EntryData(NT_Entry handle, SubscriberData* subscriber) + : handle{handle}, topic{subscriber->topic}, subscriber{subscriber} {} + + // invariants + wpi::SignalObject handle; + TopicData* topic; + SubscriberData* subscriber; + + // the publisher (created on demand) + PublisherData* publisher{nullptr}; +}; + +} // namespace nt::local diff --git a/ntcore/src/main/native/cpp/local/LocalListener.h b/ntcore/src/main/native/cpp/local/LocalListener.h new file mode 100644 index 00000000000..79cff937ce9 --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalListener.h @@ -0,0 +1,35 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include "ntcore_c.h" + +namespace nt::local { + +struct MultiSubscriberData; +struct SubscriberData; + +struct ListenerData { + ListenerData(NT_Listener handle, SubscriberData* subscriber, + unsigned int eventMask, bool subscriberOwned) + : handle{handle}, + eventMask{eventMask}, + subscriber{subscriber}, + subscriberOwned{subscriberOwned} {} + ListenerData(NT_Listener handle, MultiSubscriberData* subscriber, + unsigned int eventMask, bool subscriberOwned) + : handle{handle}, + eventMask{eventMask}, + multiSubscriber{subscriber}, + subscriberOwned{subscriberOwned} {} + + NT_Listener handle; + unsigned int eventMask; + SubscriberData* subscriber{nullptr}; + MultiSubscriberData* multiSubscriber{nullptr}; + bool subscriberOwned; +}; + +} // namespace nt::local diff --git a/ntcore/src/main/native/cpp/local/LocalMultiSubscriber.h b/ntcore/src/main/native/cpp/local/LocalMultiSubscriber.h new file mode 100644 index 00000000000..14c0369747b --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalMultiSubscriber.h @@ -0,0 +1,59 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include "Handle.h" +#include "PubSubOptions.h" +#include "VectorSet.h" +#include "ntcore_c.h" + +namespace nt::local { + +constexpr bool PrefixMatch(std::string_view name, std::string_view prefix, + bool special) { + return (!special || !prefix.empty()) && wpi::starts_with(name, prefix); +} + +struct MultiSubscriberData { + static constexpr auto kType = Handle::kMultiSubscriber; + + MultiSubscriberData(NT_MultiSubscriber handle, + std::span prefixes, + const PubSubOptionsImpl& options) + : handle{handle}, options{options} { + this->options.prefixMatch = true; + this->prefixes.reserve(prefixes.size()); + for (auto&& prefix : prefixes) { + this->prefixes.emplace_back(prefix); + } + } + + bool Matches(std::string_view name, bool special) { + for (auto&& prefix : prefixes) { + if (PrefixMatch(name, prefix, special)) { + return true; + } + } + return false; + } + + // invariants + wpi::SignalObject handle; + std::vector prefixes; + PubSubOptionsImpl options; + + // value listeners + VectorSet valueListeners; +}; + +} // namespace nt::local diff --git a/ntcore/src/main/native/cpp/local/LocalPublisher.h b/ntcore/src/main/native/cpp/local/LocalPublisher.h new file mode 100644 index 00000000000..fbdb1e5d528 --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalPublisher.h @@ -0,0 +1,36 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include + +#include "Handle.h" +#include "local/LocalTopic.h" +#include "local/PubSubConfig.h" + +namespace nt::local { + +struct PublisherData { + static constexpr auto kType = Handle::kPublisher; + + PublisherData(NT_Publisher handle, TopicData* topic, PubSubConfig config) + : handle{handle}, topic{topic}, config{std::move(config)} {} + + void UpdateActive() { + active = config.type == topic->type && config.typeStr == topic->typeStr; + } + + // invariants + wpi::SignalObject handle; + TopicData* topic; + PubSubConfig config; + + // whether or not the publisher should actually publish values + bool active{false}; +}; + +} // namespace nt::local diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp new file mode 100644 index 00000000000..590a1f32174 --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp @@ -0,0 +1,1259 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "LocalStorageImpl.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "IListenerStorage.h" +#include "Log.h" +#include "net/MessageHandler.h" + +using namespace nt; +using namespace nt::local; + +// maximum number of local publishers / subscribers to any given topic +static constexpr size_t kMaxPublishers = 512; +static constexpr size_t kMaxSubscribers = 512; +static constexpr size_t kMaxMultiSubscribers = 512; +static constexpr size_t kMaxListeners = 512; + +StorageImpl::StorageImpl(int inst, IListenerStorage& listenerStorage, + wpi::Logger& logger) + : m_inst{inst}, m_listenerStorage{listenerStorage}, m_logger{logger} {} + +// +// Network interface functions +// + +void StorageImpl::NetworkAnnounce(TopicData* topic, std::string_view typeStr, + const wpi::json& properties, + std::optional pubuid) { + DEBUG4("LS NetworkAnnounce({}, {}, {}, {})", topic->name, typeStr, + properties.dump(), pubuid.value_or(-1)); + if (pubuid.has_value()) { + return; // ack of our publish; ignore + } + + unsigned int event = NT_EVENT_NONE; + // fresh non-local publish; the network publish always sets the type even + // if it was locally published, but output a diagnostic for this case + bool didExist = topic->Exists(); + topic->onNetwork = true; + NT_Type type = StringToType(typeStr); + if (topic->type != type || topic->typeStr != typeStr) { + if (didExist) { + INFO( + "network announce of '{}' overriding local publish (was '{}', now " + "'{}')", + topic->name, topic->typeStr, typeStr); + } + topic->type = type; + topic->typeStr = typeStr; + RefreshPubSubActive(topic, true); + } + if (!didExist) { + event |= NT_EVENT_PUBLISH; + } + + // may be properties update, but need to compare to see if it actually + // changed to determine whether to update string / send event + wpi::json update = wpi::json::object(); + // added/changed + for (auto&& prop : properties.items()) { + auto it = topic->properties.find(prop.key()); + if (it == topic->properties.end() || *it != prop.value()) { + update[prop.key()] = prop.value(); + } + } + // removed + for (auto&& prop : topic->properties.items()) { + if (properties.find(prop.key()) == properties.end()) { + update[prop.key()] = wpi::json(); + } + } + if (!update.empty()) { + topic->properties = properties; + PropertiesUpdated(topic, update, event, false); + } else if (event != NT_EVENT_NONE) { + NotifyTopic(topic, event); + } +} + +void StorageImpl::RemoveNetworkPublisher(TopicData* topic) { + DEBUG4("LS RemoveNetworkPublisher({}, {})", topic->handle.GetHandle(), + topic->name); + // this acts as an unpublish + bool didExist = topic->Exists(); + topic->onNetwork = false; + if (didExist && !topic->Exists()) { + DEBUG4("Unpublished {}", topic->name); + CheckReset(topic); + NotifyTopic(topic, NT_EVENT_UNPUBLISH); + } + + if (!topic->localPublishers.empty()) { + // some other publisher still exists; if it has a different type, refresh + // and publish it over the network + auto& nextPub = topic->localPublishers.front(); + if (nextPub->config.type != topic->type || + nextPub->config.typeStr != topic->typeStr) { + topic->type = nextPub->config.type; + topic->typeStr = nextPub->config.typeStr; + RefreshPubSubActive(topic, false); + // this may result in a duplicate publish warning on the server side, + // but send one anyway in this case just to be sure + if (nextPub->active && m_network) { + m_network->ClientPublish(Handle{nextPub->handle}.GetIndex(), + topic->name, topic->typeStr, topic->properties, + nextPub->config); + } + } + } +} + +void StorageImpl::NetworkPropertiesUpdate(TopicData* topic, + const wpi::json& update, bool ack) { + DEBUG4("NetworkPropertiesUpdate({},{})", topic->name, ack); + if (ack) { + return; // ignore acks + } + SetProperties(topic, update, false); +} + +void StorageImpl::StartNetwork(net::ClientMessageHandler* network) { + DEBUG4("StartNetwork()"); + m_network = network; + // publish all active publishers to the network and send last values + // only send value once per topic + for (auto&& topic : m_topics) { + PublisherData* anyPublisher = nullptr; + for (auto&& publisher : topic->localPublishers) { + if (publisher->active) { + network->ClientPublish(Handle{publisher->handle}.GetIndex(), + topic->name, topic->typeStr, topic->properties, + publisher->config); + anyPublisher = publisher; + } + } + if (anyPublisher && topic->lastValue) { + network->ClientSetValue(Handle{anyPublisher->handle}.GetIndex(), + topic->lastValue); + } + } + for (auto&& subscriber : m_subscribers) { + if (!subscriber->config.hidden) { + network->ClientSubscribe(1 + Handle{subscriber->handle}.GetIndex(), + {{subscriber->topic->name}}, subscriber->config); + } + } + for (auto&& subscriber : m_multiSubscribers) { + if (!subscriber->options.hidden) { + network->ClientSubscribe(-1 - Handle{subscriber->handle}.GetIndex(), + subscriber->prefixes, subscriber->options); + } + } +} + +void StorageImpl::ClearNetwork() { + DEBUG4("ClearNetwork()"); + m_network = nullptr; + // treat as an unannounce all from the network side + for (auto&& topic : m_topics) { + RemoveNetworkPublisher(topic.get()); + } +} + +// +// Topic functions +// + +TopicData* StorageImpl::GetOrCreateTopic(std::string_view name) { + auto& topic = m_nameTopics[name]; + // create if it does not already exist + if (!topic) { + topic = m_topics.Add(m_inst, name); + // attach multi-subscribers + for (auto&& sub : m_multiSubscribers) { + if (sub->Matches(name, topic->special)) { + topic->multiSubscribers.Add(sub.get()); + } + } + } + return topic; +} + +// +// Topic property functions +// + +void StorageImpl::SetFlags(TopicData* topic, unsigned int flags) { + wpi::json update = wpi::json::object(); + if ((flags & NT_PERSISTENT) != 0) { + topic->properties["persistent"] = true; + update["persistent"] = true; + } else { + topic->properties.erase("persistent"); + update["persistent"] = wpi::json(); + } + if ((flags & NT_RETAINED) != 0) { + topic->properties["retained"] = true; + update["retained"] = true; + } else { + topic->properties.erase("retained"); + update["retained"] = wpi::json(); + } + if ((flags & NT_UNCACHED) != 0) { + topic->properties["cached"] = false; + update["cached"] = false; + } else { + topic->properties.erase("cached"); + update["cached"] = wpi::json(); + } + if ((flags & NT_UNCACHED) != 0) { + topic->lastValue = {}; + topic->lastValueNetwork = {}; + topic->lastValueFromNetwork = false; + } + if ((flags & NT_UNCACHED) != 0 && (flags & NT_PERSISTENT) != 0) { + WARN("topic {}: disabling cached property disables persistent storage", + topic->name); + } + topic->flags = flags; + if (!update.empty()) { + PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); + } +} + +void StorageImpl::SetPersistent(TopicData* topic, bool value) { + wpi::json update = wpi::json::object(); + if (value) { + topic->flags |= NT_PERSISTENT; + topic->properties["persistent"] = true; + update["persistent"] = true; + } else { + topic->flags &= ~NT_PERSISTENT; + topic->properties.erase("persistent"); + update["persistent"] = wpi::json(); + } + PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); +} + +void StorageImpl::SetRetained(TopicData* topic, bool value) { + wpi::json update = wpi::json::object(); + if (value) { + topic->flags |= NT_RETAINED; + topic->properties["retained"] = true; + update["retained"] = true; + } else { + topic->flags &= ~NT_RETAINED; + topic->properties.erase("retained"); + update["retained"] = wpi::json(); + } + PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); +} + +void StorageImpl::SetCached(TopicData* topic, bool value) { + wpi::json update = wpi::json::object(); + if (value) { + topic->flags &= ~NT_UNCACHED; + topic->properties.erase("cached"); + update["cached"] = wpi::json(); + } else { + topic->flags |= NT_UNCACHED; + topic->properties["cached"] = false; + update["cached"] = false; + } + PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); +} + +void StorageImpl::SetProperty(TopicData* topic, std::string_view name, + const wpi::json& value) { + if (value.is_null()) { + topic->properties.erase(name); + } else { + topic->properties[name] = value; + } + wpi::json update = wpi::json::object(); + update[name] = value; + PropertiesUpdated(topic, update, NT_EVENT_NONE, true); +} + +bool StorageImpl::SetProperties(TopicData* topic, const wpi::json& update, + bool sendNetwork) { + if (!update.is_object()) { + return false; + } + DEBUG4("SetProperties({},{})", topic->name, sendNetwork); + for (auto&& change : update.items()) { + if (change.value().is_null()) { + topic->properties.erase(change.key()); + } else { + topic->properties[change.key()] = change.value(); + } + } + PropertiesUpdated(topic, update, NT_EVENT_NONE, sendNetwork); + return true; +} + +void StorageImpl::DeleteProperty(TopicData* topic, std::string_view name) { + topic->properties.erase(name); + wpi::json update = wpi::json::object(); + update[name] = wpi::json(); + PropertiesUpdated(topic, update, NT_EVENT_NONE, true); +} + +// +// Value functions +// + +bool StorageImpl::SetEntryValue(NT_Handle pubentryHandle, const Value& value) { + if (!value) { + return false; + } + auto publisher = m_publishers.Get(pubentryHandle); + if (!publisher) { + if (auto entry = m_entries.Get(pubentryHandle)) { + publisher = PublishEntry(entry, value.type()); + } + if (!publisher) { + return false; + } + } + return PublishLocalValue(publisher, value); +} + +bool StorageImpl::SetDefaultEntryValue(NT_Handle pubsubentryHandle, + const Value& value) { + DEBUG4("SetDefaultEntryValue({}, {})", pubsubentryHandle, + static_cast(value.type())); + if (!value) { + return false; + } + if (auto topic = GetTopic(pubsubentryHandle)) { + if (!topic->IsCached()) { + WARN("ignoring default value on non-cached topic '{}'", topic->name); + return false; + } + if (!topic->lastValue && + (topic->type == NT_UNASSIGNED || topic->type == value.type() || + IsNumericCompatible(topic->type, value.type()))) { + // publish if we haven't yet + auto publisher = m_publishers.Get(pubsubentryHandle); + if (!publisher) { + if (auto entry = m_entries.Get(pubsubentryHandle)) { + publisher = PublishEntry(entry, value.type()); + } + } + + // force value timestamps to 0 + if (topic->type == NT_UNASSIGNED) { + topic->type = value.type(); + } + Value newValue; + if (topic->type == value.type()) { + newValue = value; + } else if (IsNumericCompatible(topic->type, value.type())) { + newValue = ConvertNumericValue(value, topic->type); + } else { + return true; + } + newValue.SetTime(0); + newValue.SetServerTime(0); + if (publisher) { + PublishLocalValue(publisher, newValue, true); + } else { + topic->lastValue = newValue; + } + return true; + } + } + return false; +} + +// +// Publish/Subscribe/Entry functions +// + +SubscriberData* StorageImpl::Subscribe(TopicData* topic, NT_Type type, + std::string_view typeStr, + const PubSubOptions& options) { + if (topic->localSubscribers.size() >= kMaxSubscribers) { + WPI_ERROR(m_logger, + "reached maximum number of subscribers to '{}', not subscribing", + topic->name); + return nullptr; + } + + // Create subscriber + return AddLocalSubscriber(topic, PubSubConfig{type, typeStr, options}); +} + +PublisherData* StorageImpl::Publish(TopicData* topic, NT_Type type, + std::string_view typeStr, + const wpi::json& properties, + const PubSubOptions& options) { + if (type == NT_UNASSIGNED || typeStr.empty()) { + WPI_ERROR( + m_logger, + "cannot publish '{}' with an unassigned type or empty type string", + topic->name); + return nullptr; + } + + if (topic->localPublishers.size() >= kMaxPublishers) { + WPI_ERROR(m_logger, + "reached maximum number of publishers to '{}', not publishing", + topic->name); + return nullptr; + } + + return AddLocalPublisher(topic, properties, + PubSubConfig{type, typeStr, options}); +} + +EntryData* StorageImpl::GetEntry(TopicData* topic, NT_Type type, + std::string_view typeStr, + const PubSubOptions& options) { + if (topic->localSubscribers.size() >= kMaxSubscribers) { + WPI_ERROR( + m_logger, + "reached maximum number of subscribers to '{}', not creating entry", + topic->name); + return nullptr; + } + + // Create subscriber + auto subscriber = + AddLocalSubscriber(topic, PubSubConfig{type, typeStr, options}); + + // Create entry + return AddEntry(subscriber); +} + +TopicData* StorageImpl::GetEntry(std::string_view name) { + if (name.empty()) { + return nullptr; + } + + // Get the topic data + auto* topic = GetOrCreateTopic(name); + + if (topic->entry == 0) { + if (topic->localSubscribers.size() >= kMaxSubscribers) { + WPI_ERROR( + m_logger, + "reached maximum number of subscribers to '{}', not creating entry", + topic->name); + return nullptr; + } + + // Create subscriber + auto* subscriber = AddLocalSubscriber(topic, {}); + + // Create entry + topic->entry = AddEntry(subscriber)->handle; + } + + return topic; +} + +void StorageImpl::RemoveSubEntry(NT_Handle subentryHandle) { + Handle h{subentryHandle}; + if (h.IsType(Handle::kSubscriber)) { + RemoveLocalSubscriber(subentryHandle); + } else if (h.IsType(Handle::kMultiSubscriber)) { + RemoveMultiSubscriber(subentryHandle); + } else if (h.IsType(Handle::kEntry)) { + if (auto entry = RemoveEntry(subentryHandle)) { + RemoveLocalSubscriber(entry->subscriber->handle); + if (entry->publisher) { + RemoveLocalPublisher(entry->publisher->handle); + } + } + } +} + +std::unique_ptr StorageImpl::RemoveLocalPublisher( + NT_Publisher pubHandle) { + auto publisher = m_publishers.Remove(pubHandle); + if (publisher) { + auto topic = publisher->topic; + bool didExist = topic->Exists(); + topic->localPublishers.Remove(publisher.get()); + if (didExist && !topic->Exists()) { + CheckReset(topic); + NotifyTopic(topic, NT_EVENT_UNPUBLISH); + } + + if (publisher->active && m_network) { + m_network->ClientUnpublish(Handle{publisher->handle}.GetIndex()); + } + + if (publisher->active && !topic->localPublishers.empty()) { + // some other publisher still exists; if it has a different type, refresh + // and publish it over the network + auto& nextPub = topic->localPublishers.front(); + if (nextPub->config.type != topic->type || + nextPub->config.typeStr != topic->typeStr) { + topic->type = nextPub->config.type; + topic->typeStr = nextPub->config.typeStr; + RefreshPubSubActive(topic, false); + if (nextPub->active && m_network) { + m_network->ClientPublish(Handle{nextPub->handle}.GetIndex(), + topic->name, topic->typeStr, + topic->properties, nextPub->config); + } + } + } + } + return publisher; +} + +// +// Multi-subscriber functions +// + +MultiSubscriberData* StorageImpl::AddMultiSubscriber( + std::span prefixes, const PubSubOptions& options) { + DEBUG4("AddMultiSubscriber({})", fmt::join(prefixes, ",")); + if (m_multiSubscribers.size() >= kMaxMultiSubscribers) { + WPI_ERROR(m_logger, + "reached maximum number of multi-subscribers, not subscribing"); + return nullptr; + } + auto subscriber = m_multiSubscribers.Add(m_inst, prefixes, options); + // subscribe to any already existing topics + for (auto&& topic : m_topics) { + for (auto&& prefix : prefixes) { + if (PrefixMatch(topic->name, prefix, topic->special)) { + topic->multiSubscribers.Add(subscriber); + break; + } + } + } + if (m_network && !subscriber->options.hidden) { + DEBUG4("-> NetworkSubscribe"); + m_network->ClientSubscribe(-1 - Handle{subscriber->handle}.GetIndex(), + subscriber->prefixes, subscriber->options); + } + return subscriber; +} + +std::unique_ptr StorageImpl::RemoveMultiSubscriber( + NT_MultiSubscriber subHandle) { + auto subscriber = m_multiSubscribers.Remove(subHandle); + if (subscriber) { + for (auto&& topic : m_topics) { + topic->multiSubscribers.Remove(subscriber.get()); + } + for (auto&& listener : m_listeners) { + if (listener.getSecond()->multiSubscriber == subscriber.get()) { + listener.getSecond()->multiSubscriber = nullptr; + } + } + if (m_network && !subscriber->options.hidden) { + m_network->ClientUnsubscribe(-1 - Handle{subscriber->handle}.GetIndex()); + } + } + return subscriber; +} + +// +// Lookup functions +// + +TopicData* StorageImpl::GetTopic(NT_Handle handle) { + switch (Handle{handle}.GetType()) { + case Handle::kEntry: { + if (auto entry = m_entries.Get(handle)) { + return entry->topic; + } + break; + } + case Handle::kSubscriber: { + if (auto subscriber = m_subscribers.Get(handle)) { + return subscriber->topic; + } + break; + } + case Handle::kPublisher: { + if (auto publisher = m_publishers.Get(handle)) { + return publisher->topic; + } + break; + } + case Handle::kTopic: + return m_topics.Get(handle); + default: + break; + } + return {}; +} + +SubscriberData* StorageImpl::GetSubEntry(NT_Handle subentryHandle) { + Handle h{subentryHandle}; + if (h.IsType(Handle::kSubscriber)) { + return m_subscribers.Get(subentryHandle); + } else if (h.IsType(Handle::kEntry)) { + auto entry = m_entries.Get(subentryHandle); + return entry ? entry->subscriber : nullptr; + } else { + return nullptr; + } +} + +// +// Listener functions +// + +void StorageImpl::AddListenerImpl(NT_Listener listenerHandle, TopicData* topic, + unsigned int eventMask) { + if (topic->localSubscribers.size() >= kMaxSubscribers) { + ERR("reached maximum number of subscribers to '{}', ignoring listener add", + topic->name); + return; + } + // subscribe to make sure topic updates are received + PubSubConfig config; + config.topicsOnly = (eventMask & NT_EVENT_VALUE_ALL) == 0; + auto sub = AddLocalSubscriber(topic, config); + AddListenerImpl(listenerHandle, sub, eventMask, sub->handle, true); +} + +void StorageImpl::AddListenerImpl(NT_Listener listenerHandle, + SubscriberData* subscriber, + unsigned int eventMask, + NT_Handle subentryHandle, + bool subscriberOwned) { + m_listeners.try_emplace(listenerHandle, std::make_unique( + listenerHandle, subscriber, + eventMask, subscriberOwned)); + + auto topic = subscriber->topic; + + if ((eventMask & NT_EVENT_TOPIC) != 0) { + if (topic->listeners.size() >= kMaxListeners) { + ERR("reached maximum number of listeners to '{}', not adding listener", + topic->name); + return; + } + + m_listenerStorage.Activate( + listenerHandle, eventMask & (NT_EVENT_TOPIC | NT_EVENT_IMMEDIATE)); + + topic->listeners.Add(listenerHandle); + + // handle immediate publish + if ((eventMask & (NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE)) == + (NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE) && + topic->Exists()) { + m_listenerStorage.Notify({&listenerHandle, 1}, + NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE, + topic->GetTopicInfo()); + } + } + + if ((eventMask & NT_EVENT_VALUE_ALL) != 0) { + if (subscriber->valueListeners.size() >= kMaxListeners) { + ERR("reached maximum number of listeners to '{}', not adding listener", + topic->name); + return; + } + m_listenerStorage.Activate( + listenerHandle, eventMask & (NT_EVENT_VALUE_ALL | NT_EVENT_IMMEDIATE), + [subentryHandle](unsigned int mask, Event* event) { + if (auto valueData = event->GetValueEventData()) { + valueData->subentry = subentryHandle; + } + return true; + }); + + subscriber->valueListeners.Add(listenerHandle); + + // handle immediate value + if ((eventMask & NT_EVENT_VALUE_ALL) != 0 && + (eventMask & NT_EVENT_IMMEDIATE) != 0 && topic->lastValue) { + m_listenerStorage.Notify({&listenerHandle, 1}, + NT_EVENT_IMMEDIATE | NT_EVENT_VALUE_ALL, + topic->handle, subentryHandle, topic->lastValue); + } + } +} + +void StorageImpl::AddListenerImpl(NT_Listener listenerHandle, + MultiSubscriberData* subscriber, + unsigned int eventMask, + bool subscriberOwned) { + auto listener = + m_listeners + .try_emplace(listenerHandle, std::make_unique( + listenerHandle, subscriber, + eventMask, subscriberOwned)) + .first->getSecond() + .get(); + + // if we're doing anything immediate, get the list of matching topics + wpi::SmallVector topics; + if ((eventMask & NT_EVENT_IMMEDIATE) != 0 && + (eventMask & (NT_EVENT_PUBLISH | NT_EVENT_VALUE_ALL)) != 0) { + for (auto&& topic : m_topics) { + if (topic->Exists() && subscriber->Matches(topic->name, topic->special)) { + topics.emplace_back(topic.get()); + } + } + } + + if ((eventMask & NT_EVENT_TOPIC) != 0) { + if (m_topicPrefixListeners.size() >= kMaxListeners) { + ERR("reached maximum number of listeners, not adding listener"); + return; + } + + m_listenerStorage.Activate( + listenerHandle, eventMask & (NT_EVENT_TOPIC | NT_EVENT_IMMEDIATE)); + + m_topicPrefixListeners.Add(listener); + + // handle immediate publish + if ((eventMask & (NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE)) == + (NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE)) { + std::vector topicInfos; + for (auto&& topic : topics) { + topicInfos.emplace_back(topic->GetTopicInfo()); + } + if (!topicInfos.empty()) { + m_listenerStorage.Notify({&listenerHandle, 1}, + NT_EVENT_PUBLISH | NT_EVENT_IMMEDIATE, + topicInfos); + } + } + } + + if ((eventMask & NT_EVENT_VALUE_ALL) != 0) { + if (subscriber->valueListeners.size() >= kMaxListeners) { + ERR("reached maximum number of listeners, not adding listener"); + return; + } + + m_listenerStorage.Activate( + listenerHandle, eventMask & (NT_EVENT_VALUE_ALL | NT_EVENT_IMMEDIATE), + [subentryHandle = subscriber->handle.GetHandle()](unsigned int mask, + Event* event) { + if (auto valueData = event->GetValueEventData()) { + valueData->subentry = subentryHandle; + } + return true; + }); + + subscriber->valueListeners.Add(listenerHandle); + + // handle immediate value + if ((eventMask & NT_EVENT_VALUE_ALL) != 0 && + (eventMask & NT_EVENT_IMMEDIATE) != 0) { + for (auto&& topic : topics) { + if (topic->lastValue) { + m_listenerStorage.Notify( + {&listenerHandle, 1}, NT_EVENT_VALUE_ALL | NT_EVENT_IMMEDIATE, + topic->handle, subscriber->handle, topic->lastValue); + } + } + } + } +} + +void StorageImpl::RemoveListener(NT_Listener listenerHandle, + unsigned int mask) { + auto listenerIt = m_listeners.find(listenerHandle); + if (listenerIt == m_listeners.end()) { + return; + } + auto listener = std::move(listenerIt->getSecond()); + m_listeners.erase(listenerIt); + if (!listener) { + return; + } + + m_topicPrefixListeners.Remove(listener.get()); + if (listener->subscriber) { + listener->subscriber->valueListeners.Remove(listenerHandle); + listener->subscriber->topic->listeners.Remove(listenerHandle); + if (listener->subscriberOwned) { + RemoveLocalSubscriber(listener->subscriber->handle); + } + } + if (listener->multiSubscriber) { + listener->multiSubscriber->valueListeners.Remove(listenerHandle); + if (listener->subscriberOwned) { + RemoveMultiSubscriber(listener->multiSubscriber->handle); + } + } +} + +// +// Data log functions +// + +DataLoggerData* StorageImpl::StartDataLog(wpi::log::DataLog& log, + std::string_view prefix, + std::string_view logPrefix) { + auto datalogger = m_dataloggers.Add(m_inst, log, prefix, logPrefix); + + // start logging any matching topics + auto now = nt::Now(); + for (auto&& topic : m_topics) { + if (!PrefixMatch(topic->name, prefix, topic->special) || + topic->type == NT_UNASSIGNED || topic->typeStr.empty()) { + continue; + } + topic->datalogs.emplace_back(log, datalogger->Start(topic.get(), now), + datalogger->handle); + topic->datalogType = topic->type; + + // log current value, if any + if (topic->lastValue) { + topic->datalogs.back().Append(topic->lastValue); + } + } + + return datalogger; +} + +void StorageImpl::StopDataLog(NT_DataLogger logger) { + if (auto datalogger = m_dataloggers.Remove(logger)) { + // finish any active entries + auto now = Now(); + for (auto&& topic : m_topics) { + auto it = + std::find_if(topic->datalogs.begin(), topic->datalogs.end(), + [&](const auto& elem) { return elem.logger == logger; }); + if (it != topic->datalogs.end()) { + it->log->Finish(it->entry, now); + topic->datalogs.erase(it); + } + } + } +} + +// +// Schema functions +// + +bool StorageImpl::HasSchema(std::string_view name) { + wpi::SmallString<128> fullName{"/.schema/"}; + fullName += name; + auto it = m_schemas.find(fullName); + return it != m_schemas.end(); +} + +void StorageImpl::AddSchema(std::string_view name, std::string_view type, + std::span schema) { + wpi::SmallString<128> fullName{"/.schema/"}; + fullName += name; + auto& pubHandle = m_schemas[fullName]; + if (pubHandle != 0) { + return; + } + + auto topic = GetOrCreateTopic(fullName); + + if (topic->localPublishers.size() >= kMaxPublishers) { + WPI_ERROR(m_logger, + "reached maximum number of publishers to '{}', not publishing", + topic->name); + return; + } + + pubHandle = AddLocalPublisher(topic, {{"retained", true}}, + PubSubConfig{NT_RAW, type, {}}) + ->handle; + + SetDefaultEntryValue(pubHandle, Value::MakeRaw(schema)); +} + +void StorageImpl::Reset() { + m_network = nullptr; + m_topics.clear(); + m_publishers.clear(); + m_subscribers.clear(); + m_entries.clear(); + m_multiSubscribers.clear(); + m_dataloggers.clear(); + m_nameTopics.clear(); + m_listeners.clear(); + m_topicPrefixListeners.clear(); +} + +void StorageImpl::NotifyTopic(TopicData* topic, unsigned int eventFlags) { + DEBUG4("NotifyTopic({}, {})", topic->name, eventFlags); + auto topicInfo = topic->GetTopicInfo(); + if (!topic->listeners.empty()) { + m_listenerStorage.Notify(topic->listeners, eventFlags, topicInfo); + } + + wpi::SmallVector listeners; + for (auto listener : m_topicPrefixListeners) { + if (listener->multiSubscriber && + listener->multiSubscriber->Matches(topic->name, topic->special)) { + listeners.emplace_back(listener->handle); + } + } + if (!listeners.empty()) { + m_listenerStorage.Notify(listeners, eventFlags, topicInfo); + } + + if ((eventFlags & (NT_EVENT_PUBLISH | NT_EVENT_UNPUBLISH)) != 0) { + if (!m_dataloggers.empty()) { + auto now = Now(); + for (auto&& datalogger : m_dataloggers) { + if (PrefixMatch(topic->name, datalogger->prefix, topic->special)) { + auto it = std::find_if(topic->datalogs.begin(), topic->datalogs.end(), + [&](const auto& elem) { + return elem.logger == datalogger->handle; + }); + if ((eventFlags & NT_EVENT_PUBLISH) != 0 && + it == topic->datalogs.end()) { + topic->datalogs.emplace_back(datalogger->log, + datalogger->Start(topic, now), + datalogger->handle); + topic->datalogType = topic->type; + } else if ((eventFlags & NT_EVENT_UNPUBLISH) != 0 && + it != topic->datalogs.end()) { + it->log->Finish(it->entry, now); + topic->datalogType = NT_UNASSIGNED; + topic->datalogs.erase(it); + } + } + } + } + } else if ((eventFlags & NT_EVENT_PROPERTIES) != 0) { + if (!topic->datalogs.empty()) { + auto metadata = DataLoggerEntry::MakeMetadata(topic->propertiesStr); + for (auto&& datalog : topic->datalogs) { + datalog.log->SetMetadata(datalog.entry, metadata); + } + } + } +} + +void StorageImpl::CheckReset(TopicData* topic) { + if (topic->Exists()) { + return; + } + topic->lastValue = {}; + topic->lastValueNetwork = {}; + topic->lastValueFromNetwork = false; + topic->type = NT_UNASSIGNED; + topic->typeStr.clear(); + topic->flags = 0; + topic->properties = wpi::json::object(); + topic->propertiesStr = "{}"; +} + +bool StorageImpl::SetValue(TopicData* topic, const Value& value, + unsigned int eventFlags, bool suppressIfDuplicate, + const PublisherData* publisher) { + const bool isDuplicate = topic->IsCached() && topic->lastValue == value; + DEBUG4("SetValue({}, {}, {}, {})", topic->name, value.time(), eventFlags, + isDuplicate); + if (topic->type != NT_UNASSIGNED && topic->type != value.type()) { + return false; + } + // Make sure value isn't older than last value + if (!topic->lastValue || topic->lastValue.time() == 0 || + value.time() >= topic->lastValue.time()) { + // TODO: notify option even if older value + if (!(suppressIfDuplicate && isDuplicate)) { + topic->type = value.type(); + if (topic->IsCached()) { + topic->lastValue = value; + topic->lastValueFromNetwork = false; + } + NotifyValue(topic, value, eventFlags, isDuplicate, publisher); + if (topic->datalogType == value.type()) { + for (auto&& datalog : topic->datalogs) { + datalog.Append(value); + } + } + } + } + + return true; +} + +void StorageImpl::NotifyValue(TopicData* topic, const Value& value, + unsigned int eventFlags, bool isDuplicate, + const PublisherData* publisher) { + bool isNetwork = (eventFlags & NT_EVENT_VALUE_REMOTE) != 0; + for (auto&& subscriber : topic->localSubscribers) { + if (subscriber->active && + (subscriber->config.keepDuplicates || !isDuplicate) && + ((isNetwork && !subscriber->config.disableRemote) || + (!isNetwork && !subscriber->config.disableLocal)) && + (!publisher || (publisher && (subscriber->config.excludePublisher != + publisher->handle)))) { + subscriber->pollStorage.emplace_back(value); + subscriber->handle.Set(); + if (!subscriber->valueListeners.empty()) { + m_listenerStorage.Notify(subscriber->valueListeners, eventFlags, + topic->handle, 0, value); + } + } + } + + for (auto&& subscriber : topic->multiSubscribers) { + if (subscriber->options.keepDuplicates || !isDuplicate) { + subscriber->handle.Set(); + if (!subscriber->valueListeners.empty()) { + m_listenerStorage.Notify(subscriber->valueListeners, eventFlags, + topic->handle, 0, value); + } + } + } +} + +void StorageImpl::PropertiesUpdated(TopicData* topic, const wpi::json& update, + unsigned int eventFlags, bool sendNetwork, + bool updateFlags) { + DEBUG4("PropertiesUpdated({}, {}, {}, {}, {})", topic->name, update.dump(), + eventFlags, sendNetwork, updateFlags); + if (updateFlags) { + // set flags from properties + auto it = topic->properties.find("persistent"); + if (it != topic->properties.end()) { + if (auto val = it->get_ptr()) { + if (*val) { + topic->flags |= NT_PERSISTENT; + } else { + topic->flags &= ~NT_PERSISTENT; + } + } + } + it = topic->properties.find("retained"); + if (it != topic->properties.end()) { + if (auto val = it->get_ptr()) { + if (*val) { + topic->flags |= NT_RETAINED; + } else { + topic->flags &= ~NT_RETAINED; + } + } + } + it = topic->properties.find("cached"); + if (it != topic->properties.end()) { + if (auto val = it->get_ptr()) { + if (*val) { + topic->flags &= ~NT_UNCACHED; + } else { + topic->flags |= NT_UNCACHED; + } + } + } + + if ((topic->flags & NT_UNCACHED) != 0) { + topic->lastValue = {}; + topic->lastValueNetwork = {}; + topic->lastValueFromNetwork = false; + } + + if ((topic->flags & NT_UNCACHED) != 0 && + (topic->flags & NT_PERSISTENT) != 0) { + WARN("topic {}: disabling cached property disables persistent storage", + topic->name); + } + } + + topic->propertiesStr = topic->properties.dump(); + NotifyTopic(topic, eventFlags | NT_EVENT_PROPERTIES); + // check local flag so we don't echo back received properties changes + if (m_network && sendNetwork) { + m_network->ClientSetProperties(topic->name, update); + } +} + +void StorageImpl::RefreshPubSubActive(TopicData* topic, + bool warnOnSubMismatch) { + for (auto&& publisher : topic->localPublishers) { + publisher->UpdateActive(); + } + for (auto&& subscriber : topic->localSubscribers) { + subscriber->UpdateActive(); + if (warnOnSubMismatch && topic->Exists() && !subscriber->active) { + // warn on type mismatch + INFO( + "local subscribe to '{}' disabled due to type mismatch (wanted '{}', " + "published as '{}')", + topic->name, subscriber->config.typeStr, topic->typeStr); + } + } +} + +PublisherData* StorageImpl::AddLocalPublisher(TopicData* topic, + const wpi::json& properties, + const PubSubConfig& config) { + bool didExist = topic->Exists(); + auto publisher = m_publishers.Add(m_inst, topic, config); + topic->localPublishers.Add(publisher); + + if (!didExist) { + DEBUG4("AddLocalPublisher: setting {} type {} typestr {}", topic->name, + static_cast(config.type), config.typeStr); + // set the type to the published type + topic->type = config.type; + topic->typeStr = config.typeStr; + RefreshPubSubActive(topic, true); + + if (properties.is_null()) { + topic->properties = wpi::json::object(); + } else if (properties.is_object()) { + topic->properties = properties; + } else { + WARN("ignoring non-object properties when publishing '{}'", topic->name); + topic->properties = wpi::json::object(); + } + + if (topic->properties.empty()) { + NotifyTopic(topic, NT_EVENT_PUBLISH); + } else { + PropertiesUpdated(topic, topic->properties, NT_EVENT_PUBLISH, false); + } + } else { + // only need to update just this publisher + publisher->UpdateActive(); + if (!publisher->active) { + // warn on type mismatch + INFO( + "local publish to '{}' disabled due to type mismatch (wanted '{}', " + "currently '{}')", + topic->name, config.typeStr, topic->typeStr); + } + } + + if (publisher->active && m_network) { + m_network->ClientPublish(Handle{publisher->handle}.GetIndex(), topic->name, + topic->typeStr, topic->properties, config); + } + return publisher; +} + +SubscriberData* StorageImpl::AddLocalSubscriber(TopicData* topic, + const PubSubConfig& config) { + DEBUG4("AddLocalSubscriber({})", topic->name); + auto subscriber = m_subscribers.Add(m_inst, topic, config); + topic->localSubscribers.Add(subscriber); + // set subscriber to active if the type matches + subscriber->UpdateActive(); + if (topic->Exists() && !subscriber->active) { + // warn on type mismatch + INFO( + "local subscribe to '{}' disabled due to type mismatch (wanted '{}', " + "published as '{}')", + topic->name, config.typeStr, topic->typeStr); + } + if (m_network && !subscriber->config.hidden) { + DEBUG4("-> NetworkSubscribe({})", topic->name); + m_network->ClientSubscribe(1 + Handle{subscriber->handle}.GetIndex(), + {{topic->name}}, config); + } + + // queue current value + if (subscriber->active) { + if (!topic->lastValueFromNetwork && !config.disableLocal) { + subscriber->pollStorage.emplace_back(topic->lastValue); + subscriber->handle.Set(); + } else if (topic->lastValueFromNetwork && !config.disableRemote) { + subscriber->pollStorage.emplace_back(topic->lastValueNetwork); + subscriber->handle.Set(); + } + } + return subscriber; +} + +std::unique_ptr StorageImpl::RemoveLocalSubscriber( + NT_Subscriber subHandle) { + auto subscriber = m_subscribers.Remove(subHandle); + if (subscriber) { + auto topic = subscriber->topic; + topic->localSubscribers.Remove(subscriber.get()); + for (auto&& listener : m_listeners) { + if (listener.getSecond()->subscriber == subscriber.get()) { + listener.getSecond()->subscriber = nullptr; + } + } + if (m_network && !subscriber->config.hidden) { + m_network->ClientUnsubscribe(1 + Handle{subscriber->handle}.GetIndex()); + } + } + return subscriber; +} + +PublisherData* StorageImpl::PublishEntry(EntryData* entry, NT_Type type) { + if (entry->publisher) { + return entry->publisher; + } + if (entry->subscriber->config.type == NT_UNASSIGNED) { + auto typeStr = TypeToString(type); + entry->subscriber->config.type = type; + entry->subscriber->config.typeStr = typeStr; + } else if (entry->subscriber->config.type != type) { + if (!IsNumericCompatible(type, entry->subscriber->config.type)) { + // don't allow dynamically changing the type of an entry + auto typeStr = TypeToString(type); + ERR("cannot publish entry {} as type {}, previously subscribed as {}", + entry->topic->name, typeStr, entry->subscriber->config.typeStr); + return nullptr; + } + } + // create publisher + entry->publisher = AddLocalPublisher(entry->topic, wpi::json::object(), + entry->subscriber->config); + // exclude publisher if requested + if (entry->subscriber->config.excludeSelf) { + entry->subscriber->config.excludePublisher = entry->publisher->handle; + } + return entry->publisher; +} + +bool StorageImpl::PublishLocalValue(PublisherData* publisher, + const Value& value, bool force) { + if (!value) { + return false; + } + if (publisher->topic->type != NT_UNASSIGNED && + publisher->topic->type != value.type()) { + if (IsNumericCompatible(publisher->topic->type, value.type())) { + return PublishLocalValue( + publisher, ConvertNumericValue(value, publisher->topic->type)); + } + return false; + } + if (publisher->active) { + bool isNetworkDuplicate, suppressDuplicates; + if (force || publisher->config.keepDuplicates) { + suppressDuplicates = false; + isNetworkDuplicate = false; + } else { + suppressDuplicates = true; + isNetworkDuplicate = publisher->topic->IsCached() && + (publisher->topic->lastValueNetwork == value); + } + if (!isNetworkDuplicate && m_network) { + if (publisher->topic->IsCached()) { + publisher->topic->lastValueNetwork = value; + } + m_network->ClientSetValue(Handle{publisher->handle}.GetIndex(), value); + } + return SetValue(publisher->topic, value, NT_EVENT_VALUE_LOCAL, + suppressDuplicates, publisher); + } else { + return false; + } +} diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.h b/ntcore/src/main/native/cpp/local/LocalStorageImpl.h new file mode 100644 index 00000000000..8c9f4f6ef61 --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.h @@ -0,0 +1,313 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#include "HandleMap.h" +#include "local/LocalDataLogger.h" +#include "local/LocalEntry.h" +#include "local/LocalListener.h" +#include "local/LocalMultiSubscriber.h" +#include "local/LocalPublisher.h" +#include "local/LocalSubscriber.h" +#include "local/LocalTopic.h" +#include "ntcore_c.h" +#include "ntcore_cpp.h" + +namespace wpi { +class Logger; +} // namespace wpi + +namespace nt { +class IListenerStorage; +} // namespace nt + +namespace nt::net { +class ClientMessageHandler; +} // namespace nt::net + +namespace nt::local { + +// inner struct to protect against accidentally deadlocking on the mutex +class StorageImpl { + public: + StorageImpl(int inst, IListenerStorage& listenerStorage, wpi::Logger& logger); + + // + // Network interface functions + // + + void NetworkAnnounce(TopicData* topic, std::string_view typeStr, + const wpi::json& properties, std::optional pubuid); + void RemoveNetworkPublisher(TopicData* topic); + void NetworkPropertiesUpdate(TopicData* topic, const wpi::json& update, + bool ack); + + void ServerSetValue(TopicData* topic, const Value& value) { + if (SetValue(topic, value, NT_EVENT_VALUE_REMOTE, false, nullptr)) { + if (topic->IsCached()) { + topic->lastValueNetwork = value; + topic->lastValueFromNetwork = true; + } + } + } + + void StartNetwork(net::ClientMessageHandler* network); + void ClearNetwork(); + + // + // Topic functions + // + + TopicData* GetOrCreateTopic(std::string_view name); + + template F> + void ForEachTopic(std::string_view prefix, unsigned int types, + F&& func) const { + for (auto&& topic : m_topics) { + if (!topic->Exists()) { + continue; + } + if (!wpi::starts_with(topic->name, prefix)) { + continue; + } + if (types != 0 && (types & topic->type) == 0) { + continue; + } + func(*topic); + } + } + + template F> + void ForEachTopic(std::string_view prefix, + std::span types, F&& func) const { + for (auto&& topic : m_topics) { + if (!topic->Exists()) { + continue; + } + if (!wpi::starts_with(topic->name, prefix)) { + continue; + } + if (!types.empty()) { + bool match = false; + for (auto&& type : types) { + if (topic->typeStr == type) { + match = true; + break; + } + } + if (!match) { + continue; + } + } + func(*topic); + } + } + + // + // Topic property functions + // + + void SetFlags(TopicData* topic, unsigned int flags); + void SetPersistent(TopicData* topic, bool value); + void SetRetained(TopicData* topic, bool value); + void SetCached(TopicData* topic, bool value); + + void SetProperty(TopicData* topic, std::string_view name, + const wpi::json& value); + bool SetProperties(TopicData* topic, const wpi::json& update, + bool sendNetwork); + + void DeleteProperty(TopicData* topic, std::string_view name); + + // + // Value functions + // + + bool SetEntryValue(NT_Handle pubentryHandle, const Value& value); + bool SetDefaultEntryValue(NT_Handle pubsubentryHandle, const Value& value); + + Value* GetSubEntryValue(NT_Handle subentryHandle) { + if (auto subscriber = GetSubEntry(subentryHandle)) { + return &subscriber->topic->lastValue; + } else { + return nullptr; + } + } + + // + // Publish/Subscribe/Entry functions + // + + SubscriberData* Subscribe(TopicData* topic, NT_Type type, + std::string_view typeString, + const PubSubOptions& options); + + PublisherData* Publish(TopicData* topic, NT_Type type, + std::string_view typeStr, const wpi::json& properties, + const PubSubOptions& options); + + EntryData* GetEntry(TopicData* topicHandle, NT_Type type, + std::string_view typeStr, const PubSubOptions& options); + TopicData* GetEntry(std::string_view name); + + void RemoveSubEntry(NT_Handle subentryHandle); + + std::unique_ptr RemoveLocalPublisher(NT_Publisher pubHandle); + + // + // Multi-subscriber functions + // + + MultiSubscriberData* AddMultiSubscriber( + std::span prefixes, const PubSubOptions& options); + + std::unique_ptr RemoveMultiSubscriber( + NT_MultiSubscriber subHandle); + + // + // Lookup functions + // + + TopicData* GetTopic(NT_Handle handle); + TopicData* GetTopicByHandle(NT_Topic topicHandle) { + return m_topics.Get(topicHandle); + } + TopicData* GetTopicByName(std::string_view name) { + return m_nameTopics.lookup(name); + } + TopicData* GetTopicById(int topicId) { + return m_topics.Get(Handle{m_inst, topicId, Handle::kTopic}); + } + + SubscriberData* GetSubEntry(NT_Handle subentryHandle); + + EntryData* GetEntryByHandle(NT_Entry entryHandle) { + return m_entries.Get(entryHandle); + } + + MultiSubscriberData* GetMultiSubscriberByHandle(NT_MultiSubscriber handle) { + return m_multiSubscribers.Get(handle); + } + + SubscriberData* GetSubscriberByHandle(NT_Subscriber handle) { + return m_subscribers.Get(handle); + } + + // + // Listener functions + // + + void AddListenerImpl(NT_Listener listenerHandle, TopicData* topic, + unsigned int eventMask); + void AddListenerImpl(NT_Listener listenerHandle, SubscriberData* subscriber, + unsigned int eventMask, NT_Handle subentryHandle, + bool subscriberOwned); + void AddListenerImpl(NT_Listener listenerHandle, + MultiSubscriberData* subscriber, unsigned int eventMask, + bool subscriberOwned); + void RemoveListener(NT_Listener listener, unsigned int mask); + + // + // Data log functions + // + + DataLoggerData* StartDataLog(wpi::log::DataLog& log, std::string_view prefix, + std::string_view logPrefix); + void StopDataLog(NT_DataLogger logger); + + // + // Schema functions + // + + bool HasSchema(std::string_view name); + void AddSchema(std::string_view name, std::string_view type, + std::span schema); + + void Reset(); + + private: + // topic functions + void NotifyTopic(TopicData* topic, unsigned int eventFlags); + + void CheckReset(TopicData* topic); + + bool SetValue(TopicData* topic, const Value& value, unsigned int eventFlags, + bool suppressIfDuplicate, const PublisherData* publisher); + void NotifyValue(TopicData* topic, const Value& value, + unsigned int eventFlags, bool isDuplicate, + const PublisherData* publisher); + + void PropertiesUpdated(TopicData* topic, const wpi::json& update, + unsigned int eventFlags, bool sendNetwork, + bool updateFlags = true); + + void RefreshPubSubActive(TopicData* topic, bool warnOnSubMismatch); + + PublisherData* AddLocalPublisher(TopicData* topic, + const wpi::json& properties, + const PubSubConfig& options); + + SubscriberData* AddLocalSubscriber(TopicData* topic, + const PubSubConfig& options); + + std::unique_ptr RemoveLocalSubscriber( + NT_Subscriber subHandle); + + EntryData* AddEntry(SubscriberData* subscriber) { + auto entry = m_entries.Add(m_inst, subscriber); + subscriber->topic->entries.Add(entry); + return entry; + } + + std::unique_ptr RemoveEntry(NT_Entry entryHandle) { + auto entry = m_entries.Remove(entryHandle); + if (entry) { + entry->topic->entries.Remove(entry.get()); + } + return entry; + } + + PublisherData* PublishEntry(EntryData* entry, NT_Type type); + + bool PublishLocalValue(PublisherData* publisher, const Value& value, + bool force = false); + + private: + int m_inst; + IListenerStorage& m_listenerStorage; + wpi::Logger& m_logger; + net::ClientMessageHandler* m_network{nullptr}; + + // handle mappings + HandleMap m_topics; + HandleMap m_publishers; + HandleMap m_subscribers; + HandleMap m_entries; + HandleMap m_multiSubscribers; + HandleMap m_dataloggers; + + // name mappings + wpi::StringMap m_nameTopics; + + // listeners + wpi::DenseMap> m_listeners; + + // string-based listeners + VectorSet m_topicPrefixListeners; + + // schema publishers + wpi::StringMap m_schemas; +}; + +} // namespace nt::local diff --git a/ntcore/src/main/native/cpp/local/LocalSubscriber.h b/ntcore/src/main/native/cpp/local/LocalSubscriber.h new file mode 100644 index 00000000000..5545185831f --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalSubscriber.h @@ -0,0 +1,51 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +#include "Handle.h" +#include "Types_internal.h" +#include "ValueCircularBuffer.h" +#include "VectorSet.h" +#include "local/LocalTopic.h" +#include "local/PubSubConfig.h" +#include "ntcore_c.h" + +namespace nt::local { + +struct SubscriberData { + static constexpr auto kType = Handle::kSubscriber; + + SubscriberData(NT_Subscriber handle, TopicData* topic, PubSubConfig config) + : handle{handle}, + topic{topic}, + config{std::move(config)}, + pollStorage{config.pollStorage} {} + + void UpdateActive() { + // for subscribers, unassigned is a wildcard + // also allow numerically compatible subscribers + active = config.type == NT_UNASSIGNED || + (config.type == topic->type && config.typeStr == topic->typeStr) || + IsNumericCompatible(config.type, topic->type); + } + + // invariants + wpi::SignalObject handle; + TopicData* topic; + PubSubConfig config; + + // whether or not the subscriber should actually receive values + bool active{false}; + + // polling storage + ValueCircularBuffer pollStorage; + + // value listeners + VectorSet valueListeners; +}; + +} // namespace nt::local diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.h b/ntcore/src/main/native/cpp/local/LocalTopic.h new file mode 100644 index 00000000000..d82d0fd0462 --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalTopic.h @@ -0,0 +1,77 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +#include +#include +#include + +#include "Handle.h" +#include "VectorSet.h" +#include "local/LocalDataLoggerEntry.h" +#include "ntcore_cpp.h" + +namespace nt::local { + +struct EntryData; +struct MultiSubscriberData; +struct PublisherData; +struct SubscriberData; + +constexpr bool IsSpecial(std::string_view name) { + return name.empty() ? false : name.front() == '$'; +} + +struct TopicData { + static constexpr auto kType = Handle::kTopic; + + TopicData(NT_Topic handle, std::string_view name) + : handle{handle}, name{name}, special{IsSpecial(name)} {} + + bool Exists() const { return onNetwork || !localPublishers.empty(); } + + bool IsCached() const { return (flags & NT_UNCACHED) == 0; } + + TopicInfo GetTopicInfo() const { + TopicInfo info; + info.topic = handle; + info.name = name; + info.type = type; + info.type_str = typeStr; + info.properties = propertiesStr; + return info; + } + + // invariants + wpi::SignalObject handle; + std::string name; + bool special; + + Value lastValue; // also stores timestamp + Value lastValueNetwork; + NT_Type type{NT_UNASSIGNED}; + std::string typeStr; + unsigned int flags{0}; // for NT3 APIs + std::string propertiesStr{"{}"}; // cached string for GetTopicInfo() et al + wpi::json properties = wpi::json::object(); + NT_Entry entry{0}; // cached entry for GetEntry() + + bool onNetwork{false}; // true if there are any remote publishers + bool lastValueFromNetwork{false}; + + wpi::SmallVector datalogs; + NT_Type datalogType{NT_UNASSIGNED}; + + VectorSet localPublishers; + VectorSet localSubscribers; + VectorSet multiSubscribers; + VectorSet entries; + VectorSet listeners; +}; + +} // namespace nt::local diff --git a/ntcore/src/main/native/cpp/local/PubSubConfig.h b/ntcore/src/main/native/cpp/local/PubSubConfig.h new file mode 100644 index 00000000000..ab5c118f9bc --- /dev/null +++ b/ntcore/src/main/native/cpp/local/PubSubConfig.h @@ -0,0 +1,27 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +#include "PubSubOptions.h" +#include "ntcore_c.h" + +namespace nt::local { + +struct PubSubConfig : public PubSubOptionsImpl { + PubSubConfig() = default; + PubSubConfig(NT_Type type, std::string_view typeStr, + const PubSubOptions& options) + : PubSubOptionsImpl{options}, type{type}, typeStr{typeStr} { + prefixMatch = false; + } + + NT_Type type{NT_UNASSIGNED}; + std::string typeStr; +}; + +} // namespace nt::local From db106ccf250a0bc8d326fe43515a2cb38c38b3ae Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Fri, 18 Oct 2024 23:14:08 -0700 Subject: [PATCH 17/26] [ntcore] LocalStorage: Change GetEntry to return Entry --- ntcore/src/main/native/cpp/LocalStorage.h | 4 ++-- ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp | 8 ++++---- ntcore/src/main/native/cpp/local/LocalStorageImpl.h | 2 +- ntcore/src/main/native/cpp/local/LocalTopic.h | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ntcore/src/main/native/cpp/LocalStorage.h b/ntcore/src/main/native/cpp/LocalStorage.h index 30c8c5b2bfb..8d979791f87 100644 --- a/ntcore/src/main/native/cpp/LocalStorage.h +++ b/ntcore/src/main/native/cpp/LocalStorage.h @@ -397,8 +397,8 @@ class LocalStorage final : public net::ILocalStorage { // Index-only NT_Entry GetEntry(std::string_view name) { std::scoped_lock lock{m_mutex}; - if (auto topic = m_impl.GetEntry(name)) { - return topic->entry; + if (auto entry = m_impl.GetEntry(name)) { + return entry->handle; } else { return {}; } diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp index 590a1f32174..0186829883c 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp @@ -439,7 +439,7 @@ EntryData* StorageImpl::GetEntry(TopicData* topic, NT_Type type, return AddEntry(subscriber); } -TopicData* StorageImpl::GetEntry(std::string_view name) { +EntryData* StorageImpl::GetEntry(std::string_view name) { if (name.empty()) { return nullptr; } @@ -447,7 +447,7 @@ TopicData* StorageImpl::GetEntry(std::string_view name) { // Get the topic data auto* topic = GetOrCreateTopic(name); - if (topic->entry == 0) { + if (!topic->entry) { if (topic->localSubscribers.size() >= kMaxSubscribers) { WPI_ERROR( m_logger, @@ -460,10 +460,10 @@ TopicData* StorageImpl::GetEntry(std::string_view name) { auto* subscriber = AddLocalSubscriber(topic, {}); // Create entry - topic->entry = AddEntry(subscriber)->handle; + topic->entry = AddEntry(subscriber); } - return topic; + return topic->entry; } void StorageImpl::RemoveSubEntry(NT_Handle subentryHandle) { diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.h b/ntcore/src/main/native/cpp/local/LocalStorageImpl.h index 8c9f4f6ef61..d82424701a0 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.h +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.h @@ -159,7 +159,7 @@ class StorageImpl { EntryData* GetEntry(TopicData* topicHandle, NT_Type type, std::string_view typeStr, const PubSubOptions& options); - TopicData* GetEntry(std::string_view name); + EntryData* GetEntry(std::string_view name); void RemoveSubEntry(NT_Handle subentryHandle); diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.h b/ntcore/src/main/native/cpp/local/LocalTopic.h index d82d0fd0462..a3d5e2a3d14 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.h +++ b/ntcore/src/main/native/cpp/local/LocalTopic.h @@ -59,7 +59,7 @@ struct TopicData { unsigned int flags{0}; // for NT3 APIs std::string propertiesStr{"{}"}; // cached string for GetTopicInfo() et al wpi::json properties = wpi::json::object(); - NT_Entry entry{0}; // cached entry for GetEntry() + EntryData* entry{nullptr}; // cached entry for GetEntry() bool onNetwork{false}; // true if there are any remote publishers bool lastValueFromNetwork{false}; From 337bc1cfd087fe4d64461db4d006cd4871cbe194 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Fri, 18 Oct 2024 23:17:49 -0700 Subject: [PATCH 18/26] [ntcore] LocalStorage: Rename classes to Local* --- ntcore/src/main/native/cpp/LocalStorage.h | 2 +- .../main/native/cpp/local/LocalDataLogger.cpp | 4 +- .../main/native/cpp/local/LocalDataLogger.h | 10 +- .../native/cpp/local/LocalDataLoggerEntry.cpp | 4 +- .../native/cpp/local/LocalDataLoggerEntry.h | 6 +- ntcore/src/main/native/cpp/local/LocalEntry.h | 12 +- .../src/main/native/cpp/local/LocalListener.h | 18 +-- .../native/cpp/local/LocalMultiSubscriber.h | 8 +- .../main/native/cpp/local/LocalPublisher.h | 6 +- .../native/cpp/local/LocalStorageImpl.cpp | 108 +++++++------- .../main/native/cpp/local/LocalStorageImpl.h | 132 +++++++++--------- .../main/native/cpp/local/LocalSubscriber.h | 6 +- ntcore/src/main/native/cpp/local/LocalTopic.h | 24 ++-- 13 files changed, 170 insertions(+), 170 deletions(-) diff --git a/ntcore/src/main/native/cpp/LocalStorage.h b/ntcore/src/main/native/cpp/LocalStorage.h index 8d979791f87..17f012cd598 100644 --- a/ntcore/src/main/native/cpp/LocalStorage.h +++ b/ntcore/src/main/native/cpp/LocalStorage.h @@ -173,7 +173,7 @@ class LocalStorage final : public net::ILocalStorage { bool GetTopicExists(NT_Handle handle) { std::scoped_lock lock{m_mutex}; - local::TopicData* topic = m_impl.GetTopic(handle); + local::LocalTopic* topic = m_impl.GetTopic(handle); return topic && topic->Exists(); } diff --git a/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp b/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp index f142ec920e0..86028b2f37b 100644 --- a/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp +++ b/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp @@ -13,7 +13,7 @@ using namespace nt::local; -int DataLoggerData::Start(TopicData* topic, int64_t time) { +int LocalDataLogger::Start(LocalTopic* topic, int64_t time) { std::string_view typeStr = topic->typeStr; // NT and DataLog use different standard representations for int and int[] if (typeStr == "int") { @@ -25,5 +25,5 @@ int DataLoggerData::Start(TopicData* topic, int64_t time) { fmt::format( "{}{}", logPrefix, wpi::remove_prefix(topic->name, prefix).value_or(topic->name)), - typeStr, DataLoggerEntry::MakeMetadata(topic->propertiesStr), time); + typeStr, LocalDataLoggerEntry::MakeMetadata(topic->propertiesStr), time); } diff --git a/ntcore/src/main/native/cpp/local/LocalDataLogger.h b/ntcore/src/main/native/cpp/local/LocalDataLogger.h index 431d9346b05..66d32724300 100644 --- a/ntcore/src/main/native/cpp/local/LocalDataLogger.h +++ b/ntcore/src/main/native/cpp/local/LocalDataLogger.h @@ -18,16 +18,16 @@ class DataLog; namespace nt::local { -struct TopicData; +struct LocalTopic; -struct DataLoggerData { +struct LocalDataLogger { static constexpr auto kType = Handle::kDataLogger; - DataLoggerData(NT_DataLogger handle, wpi::log::DataLog& log, - std::string_view prefix, std::string_view logPrefix) + LocalDataLogger(NT_DataLogger handle, wpi::log::DataLog& log, + std::string_view prefix, std::string_view logPrefix) : handle{handle}, log{log}, prefix{prefix}, logPrefix{logPrefix} {} - int Start(TopicData* topic, int64_t time); + int Start(LocalTopic* topic, int64_t time); NT_DataLogger handle; wpi::log::DataLog& log; diff --git a/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp index de06f737f7f..ed5a1bba9df 100644 --- a/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp +++ b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp @@ -12,11 +12,11 @@ using namespace nt::local; -std::string DataLoggerEntry::MakeMetadata(std::string_view properties) { +std::string LocalDataLoggerEntry::MakeMetadata(std::string_view properties) { return fmt::format("{{\"properties\":{},\"source\":\"NT\"}}", properties); } -void DataLoggerEntry::Append(const Value& v) { +void LocalDataLoggerEntry::Append(const Value& v) { auto time = v.time(); switch (v.type()) { case NT_BOOLEAN: diff --git a/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h index 8b8be1a951f..605635f9ab7 100644 --- a/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h +++ b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h @@ -19,10 +19,10 @@ class Value; namespace nt::local { -struct TopicData; +struct LocalTopic; -struct DataLoggerEntry { - DataLoggerEntry(wpi::log::DataLog& log, int entry, NT_DataLogger logger) +struct LocalDataLoggerEntry { + LocalDataLoggerEntry(wpi::log::DataLog& log, int entry, NT_DataLogger logger) : log{&log}, entry{entry}, logger{logger} {} static std::string MakeMetadata(std::string_view properties); diff --git a/ntcore/src/main/native/cpp/local/LocalEntry.h b/ntcore/src/main/native/cpp/local/LocalEntry.h index 334e59a838e..01f43d475a3 100644 --- a/ntcore/src/main/native/cpp/local/LocalEntry.h +++ b/ntcore/src/main/native/cpp/local/LocalEntry.h @@ -11,21 +11,21 @@ namespace nt::local { -struct PublisherData; +struct LocalPublisher; -struct EntryData { +struct LocalEntry { static constexpr auto kType = Handle::kEntry; - EntryData(NT_Entry handle, SubscriberData* subscriber) + LocalEntry(NT_Entry handle, LocalSubscriber* subscriber) : handle{handle}, topic{subscriber->topic}, subscriber{subscriber} {} // invariants wpi::SignalObject handle; - TopicData* topic; - SubscriberData* subscriber; + LocalTopic* topic; + LocalSubscriber* subscriber; // the publisher (created on demand) - PublisherData* publisher{nullptr}; + LocalPublisher* publisher{nullptr}; }; } // namespace nt::local diff --git a/ntcore/src/main/native/cpp/local/LocalListener.h b/ntcore/src/main/native/cpp/local/LocalListener.h index 79cff937ce9..89b21fcc15c 100644 --- a/ntcore/src/main/native/cpp/local/LocalListener.h +++ b/ntcore/src/main/native/cpp/local/LocalListener.h @@ -8,18 +8,18 @@ namespace nt::local { -struct MultiSubscriberData; -struct SubscriberData; +struct LocalMultiSubscriber; +struct LocalSubscriber; -struct ListenerData { - ListenerData(NT_Listener handle, SubscriberData* subscriber, - unsigned int eventMask, bool subscriberOwned) +struct LocalListener { + LocalListener(NT_Listener handle, LocalSubscriber* subscriber, + unsigned int eventMask, bool subscriberOwned) : handle{handle}, eventMask{eventMask}, subscriber{subscriber}, subscriberOwned{subscriberOwned} {} - ListenerData(NT_Listener handle, MultiSubscriberData* subscriber, - unsigned int eventMask, bool subscriberOwned) + LocalListener(NT_Listener handle, LocalMultiSubscriber* subscriber, + unsigned int eventMask, bool subscriberOwned) : handle{handle}, eventMask{eventMask}, multiSubscriber{subscriber}, @@ -27,8 +27,8 @@ struct ListenerData { NT_Listener handle; unsigned int eventMask; - SubscriberData* subscriber{nullptr}; - MultiSubscriberData* multiSubscriber{nullptr}; + LocalSubscriber* subscriber{nullptr}; + LocalMultiSubscriber* multiSubscriber{nullptr}; bool subscriberOwned; }; diff --git a/ntcore/src/main/native/cpp/local/LocalMultiSubscriber.h b/ntcore/src/main/native/cpp/local/LocalMultiSubscriber.h index 14c0369747b..89627599179 100644 --- a/ntcore/src/main/native/cpp/local/LocalMultiSubscriber.h +++ b/ntcore/src/main/native/cpp/local/LocalMultiSubscriber.h @@ -24,12 +24,12 @@ constexpr bool PrefixMatch(std::string_view name, std::string_view prefix, return (!special || !prefix.empty()) && wpi::starts_with(name, prefix); } -struct MultiSubscriberData { +struct LocalMultiSubscriber { static constexpr auto kType = Handle::kMultiSubscriber; - MultiSubscriberData(NT_MultiSubscriber handle, - std::span prefixes, - const PubSubOptionsImpl& options) + LocalMultiSubscriber(NT_MultiSubscriber handle, + std::span prefixes, + const PubSubOptionsImpl& options) : handle{handle}, options{options} { this->options.prefixMatch = true; this->prefixes.reserve(prefixes.size()); diff --git a/ntcore/src/main/native/cpp/local/LocalPublisher.h b/ntcore/src/main/native/cpp/local/LocalPublisher.h index fbdb1e5d528..611c0387356 100644 --- a/ntcore/src/main/native/cpp/local/LocalPublisher.h +++ b/ntcore/src/main/native/cpp/local/LocalPublisher.h @@ -14,10 +14,10 @@ namespace nt::local { -struct PublisherData { +struct LocalPublisher { static constexpr auto kType = Handle::kPublisher; - PublisherData(NT_Publisher handle, TopicData* topic, PubSubConfig config) + LocalPublisher(NT_Publisher handle, LocalTopic* topic, PubSubConfig config) : handle{handle}, topic{topic}, config{std::move(config)} {} void UpdateActive() { @@ -26,7 +26,7 @@ struct PublisherData { // invariants wpi::SignalObject handle; - TopicData* topic; + LocalTopic* topic; PubSubConfig config; // whether or not the publisher should actually publish values diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp index 0186829883c..905ad302615 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp @@ -34,7 +34,7 @@ StorageImpl::StorageImpl(int inst, IListenerStorage& listenerStorage, // Network interface functions // -void StorageImpl::NetworkAnnounce(TopicData* topic, std::string_view typeStr, +void StorageImpl::NetworkAnnounce(LocalTopic* topic, std::string_view typeStr, const wpi::json& properties, std::optional pubuid) { DEBUG4("LS NetworkAnnounce({}, {}, {}, {})", topic->name, typeStr, @@ -88,7 +88,7 @@ void StorageImpl::NetworkAnnounce(TopicData* topic, std::string_view typeStr, } } -void StorageImpl::RemoveNetworkPublisher(TopicData* topic) { +void StorageImpl::RemoveNetworkPublisher(LocalTopic* topic) { DEBUG4("LS RemoveNetworkPublisher({}, {})", topic->handle.GetHandle(), topic->name); // this acts as an unpublish @@ -120,7 +120,7 @@ void StorageImpl::RemoveNetworkPublisher(TopicData* topic) { } } -void StorageImpl::NetworkPropertiesUpdate(TopicData* topic, +void StorageImpl::NetworkPropertiesUpdate(LocalTopic* topic, const wpi::json& update, bool ack) { DEBUG4("NetworkPropertiesUpdate({},{})", topic->name, ack); if (ack) { @@ -135,7 +135,7 @@ void StorageImpl::StartNetwork(net::ClientMessageHandler* network) { // publish all active publishers to the network and send last values // only send value once per topic for (auto&& topic : m_topics) { - PublisherData* anyPublisher = nullptr; + LocalPublisher* anyPublisher = nullptr; for (auto&& publisher : topic->localPublishers) { if (publisher->active) { network->ClientPublish(Handle{publisher->handle}.GetIndex(), @@ -176,7 +176,7 @@ void StorageImpl::ClearNetwork() { // Topic functions // -TopicData* StorageImpl::GetOrCreateTopic(std::string_view name) { +LocalTopic* StorageImpl::GetOrCreateTopic(std::string_view name) { auto& topic = m_nameTopics[name]; // create if it does not already exist if (!topic) { @@ -195,7 +195,7 @@ TopicData* StorageImpl::GetOrCreateTopic(std::string_view name) { // Topic property functions // -void StorageImpl::SetFlags(TopicData* topic, unsigned int flags) { +void StorageImpl::SetFlags(LocalTopic* topic, unsigned int flags) { wpi::json update = wpi::json::object(); if ((flags & NT_PERSISTENT) != 0) { topic->properties["persistent"] = true; @@ -233,7 +233,7 @@ void StorageImpl::SetFlags(TopicData* topic, unsigned int flags) { } } -void StorageImpl::SetPersistent(TopicData* topic, bool value) { +void StorageImpl::SetPersistent(LocalTopic* topic, bool value) { wpi::json update = wpi::json::object(); if (value) { topic->flags |= NT_PERSISTENT; @@ -247,7 +247,7 @@ void StorageImpl::SetPersistent(TopicData* topic, bool value) { PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); } -void StorageImpl::SetRetained(TopicData* topic, bool value) { +void StorageImpl::SetRetained(LocalTopic* topic, bool value) { wpi::json update = wpi::json::object(); if (value) { topic->flags |= NT_RETAINED; @@ -261,7 +261,7 @@ void StorageImpl::SetRetained(TopicData* topic, bool value) { PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); } -void StorageImpl::SetCached(TopicData* topic, bool value) { +void StorageImpl::SetCached(LocalTopic* topic, bool value) { wpi::json update = wpi::json::object(); if (value) { topic->flags &= ~NT_UNCACHED; @@ -275,7 +275,7 @@ void StorageImpl::SetCached(TopicData* topic, bool value) { PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); } -void StorageImpl::SetProperty(TopicData* topic, std::string_view name, +void StorageImpl::SetProperty(LocalTopic* topic, std::string_view name, const wpi::json& value) { if (value.is_null()) { topic->properties.erase(name); @@ -287,7 +287,7 @@ void StorageImpl::SetProperty(TopicData* topic, std::string_view name, PropertiesUpdated(topic, update, NT_EVENT_NONE, true); } -bool StorageImpl::SetProperties(TopicData* topic, const wpi::json& update, +bool StorageImpl::SetProperties(LocalTopic* topic, const wpi::json& update, bool sendNetwork) { if (!update.is_object()) { return false; @@ -304,7 +304,7 @@ bool StorageImpl::SetProperties(TopicData* topic, const wpi::json& update, return true; } -void StorageImpl::DeleteProperty(TopicData* topic, std::string_view name) { +void StorageImpl::DeleteProperty(LocalTopic* topic, std::string_view name) { topic->properties.erase(name); wpi::json update = wpi::json::object(); update[name] = wpi::json(); @@ -383,9 +383,9 @@ bool StorageImpl::SetDefaultEntryValue(NT_Handle pubsubentryHandle, // Publish/Subscribe/Entry functions // -SubscriberData* StorageImpl::Subscribe(TopicData* topic, NT_Type type, - std::string_view typeStr, - const PubSubOptions& options) { +LocalSubscriber* StorageImpl::Subscribe(LocalTopic* topic, NT_Type type, + std::string_view typeStr, + const PubSubOptions& options) { if (topic->localSubscribers.size() >= kMaxSubscribers) { WPI_ERROR(m_logger, "reached maximum number of subscribers to '{}', not subscribing", @@ -397,10 +397,10 @@ SubscriberData* StorageImpl::Subscribe(TopicData* topic, NT_Type type, return AddLocalSubscriber(topic, PubSubConfig{type, typeStr, options}); } -PublisherData* StorageImpl::Publish(TopicData* topic, NT_Type type, - std::string_view typeStr, - const wpi::json& properties, - const PubSubOptions& options) { +LocalPublisher* StorageImpl::Publish(LocalTopic* topic, NT_Type type, + std::string_view typeStr, + const wpi::json& properties, + const PubSubOptions& options) { if (type == NT_UNASSIGNED || typeStr.empty()) { WPI_ERROR( m_logger, @@ -420,9 +420,9 @@ PublisherData* StorageImpl::Publish(TopicData* topic, NT_Type type, PubSubConfig{type, typeStr, options}); } -EntryData* StorageImpl::GetEntry(TopicData* topic, NT_Type type, - std::string_view typeStr, - const PubSubOptions& options) { +LocalEntry* StorageImpl::GetEntry(LocalTopic* topic, NT_Type type, + std::string_view typeStr, + const PubSubOptions& options) { if (topic->localSubscribers.size() >= kMaxSubscribers) { WPI_ERROR( m_logger, @@ -439,7 +439,7 @@ EntryData* StorageImpl::GetEntry(TopicData* topic, NT_Type type, return AddEntry(subscriber); } -EntryData* StorageImpl::GetEntry(std::string_view name) { +LocalEntry* StorageImpl::GetEntry(std::string_view name) { if (name.empty()) { return nullptr; } @@ -482,7 +482,7 @@ void StorageImpl::RemoveSubEntry(NT_Handle subentryHandle) { } } -std::unique_ptr StorageImpl::RemoveLocalPublisher( +std::unique_ptr StorageImpl::RemoveLocalPublisher( NT_Publisher pubHandle) { auto publisher = m_publishers.Remove(pubHandle); if (publisher) { @@ -522,7 +522,7 @@ std::unique_ptr StorageImpl::RemoveLocalPublisher( // Multi-subscriber functions // -MultiSubscriberData* StorageImpl::AddMultiSubscriber( +LocalMultiSubscriber* StorageImpl::AddMultiSubscriber( std::span prefixes, const PubSubOptions& options) { DEBUG4("AddMultiSubscriber({})", fmt::join(prefixes, ",")); if (m_multiSubscribers.size() >= kMaxMultiSubscribers) { @@ -548,7 +548,7 @@ MultiSubscriberData* StorageImpl::AddMultiSubscriber( return subscriber; } -std::unique_ptr StorageImpl::RemoveMultiSubscriber( +std::unique_ptr StorageImpl::RemoveMultiSubscriber( NT_MultiSubscriber subHandle) { auto subscriber = m_multiSubscribers.Remove(subHandle); if (subscriber) { @@ -571,7 +571,7 @@ std::unique_ptr StorageImpl::RemoveMultiSubscriber( // Lookup functions // -TopicData* StorageImpl::GetTopic(NT_Handle handle) { +LocalTopic* StorageImpl::GetTopic(NT_Handle handle) { switch (Handle{handle}.GetType()) { case Handle::kEntry: { if (auto entry = m_entries.Get(handle)) { @@ -599,7 +599,7 @@ TopicData* StorageImpl::GetTopic(NT_Handle handle) { return {}; } -SubscriberData* StorageImpl::GetSubEntry(NT_Handle subentryHandle) { +LocalSubscriber* StorageImpl::GetSubEntry(NT_Handle subentryHandle) { Handle h{subentryHandle}; if (h.IsType(Handle::kSubscriber)) { return m_subscribers.Get(subentryHandle); @@ -615,7 +615,7 @@ SubscriberData* StorageImpl::GetSubEntry(NT_Handle subentryHandle) { // Listener functions // -void StorageImpl::AddListenerImpl(NT_Listener listenerHandle, TopicData* topic, +void StorageImpl::AddListenerImpl(NT_Listener listenerHandle, LocalTopic* topic, unsigned int eventMask) { if (topic->localSubscribers.size() >= kMaxSubscribers) { ERR("reached maximum number of subscribers to '{}', ignoring listener add", @@ -630,11 +630,11 @@ void StorageImpl::AddListenerImpl(NT_Listener listenerHandle, TopicData* topic, } void StorageImpl::AddListenerImpl(NT_Listener listenerHandle, - SubscriberData* subscriber, + LocalSubscriber* subscriber, unsigned int eventMask, NT_Handle subentryHandle, bool subscriberOwned) { - m_listeners.try_emplace(listenerHandle, std::make_unique( + m_listeners.try_emplace(listenerHandle, std::make_unique( listenerHandle, subscriber, eventMask, subscriberOwned)); @@ -690,19 +690,19 @@ void StorageImpl::AddListenerImpl(NT_Listener listenerHandle, } void StorageImpl::AddListenerImpl(NT_Listener listenerHandle, - MultiSubscriberData* subscriber, + LocalMultiSubscriber* subscriber, unsigned int eventMask, bool subscriberOwned) { auto listener = m_listeners - .try_emplace(listenerHandle, std::make_unique( + .try_emplace(listenerHandle, std::make_unique( listenerHandle, subscriber, eventMask, subscriberOwned)) .first->getSecond() .get(); // if we're doing anything immediate, get the list of matching topics - wpi::SmallVector topics; + wpi::SmallVector topics; if ((eventMask & NT_EVENT_IMMEDIATE) != 0 && (eventMask & (NT_EVENT_PUBLISH | NT_EVENT_VALUE_ALL)) != 0) { for (auto&& topic : m_topics) { @@ -802,9 +802,9 @@ void StorageImpl::RemoveListener(NT_Listener listenerHandle, // Data log functions // -DataLoggerData* StorageImpl::StartDataLog(wpi::log::DataLog& log, - std::string_view prefix, - std::string_view logPrefix) { +LocalDataLogger* StorageImpl::StartDataLog(wpi::log::DataLog& log, + std::string_view prefix, + std::string_view logPrefix) { auto datalogger = m_dataloggers.Add(m_inst, log, prefix, logPrefix); // start logging any matching topics @@ -892,7 +892,7 @@ void StorageImpl::Reset() { m_topicPrefixListeners.clear(); } -void StorageImpl::NotifyTopic(TopicData* topic, unsigned int eventFlags) { +void StorageImpl::NotifyTopic(LocalTopic* topic, unsigned int eventFlags) { DEBUG4("NotifyTopic({}, {})", topic->name, eventFlags); auto topicInfo = topic->GetTopicInfo(); if (!topic->listeners.empty()) { @@ -936,7 +936,7 @@ void StorageImpl::NotifyTopic(TopicData* topic, unsigned int eventFlags) { } } else if ((eventFlags & NT_EVENT_PROPERTIES) != 0) { if (!topic->datalogs.empty()) { - auto metadata = DataLoggerEntry::MakeMetadata(topic->propertiesStr); + auto metadata = LocalDataLoggerEntry::MakeMetadata(topic->propertiesStr); for (auto&& datalog : topic->datalogs) { datalog.log->SetMetadata(datalog.entry, metadata); } @@ -944,7 +944,7 @@ void StorageImpl::NotifyTopic(TopicData* topic, unsigned int eventFlags) { } } -void StorageImpl::CheckReset(TopicData* topic) { +void StorageImpl::CheckReset(LocalTopic* topic) { if (topic->Exists()) { return; } @@ -958,9 +958,9 @@ void StorageImpl::CheckReset(TopicData* topic) { topic->propertiesStr = "{}"; } -bool StorageImpl::SetValue(TopicData* topic, const Value& value, +bool StorageImpl::SetValue(LocalTopic* topic, const Value& value, unsigned int eventFlags, bool suppressIfDuplicate, - const PublisherData* publisher) { + const LocalPublisher* publisher) { const bool isDuplicate = topic->IsCached() && topic->lastValue == value; DEBUG4("SetValue({}, {}, {}, {})", topic->name, value.time(), eventFlags, isDuplicate); @@ -989,9 +989,9 @@ bool StorageImpl::SetValue(TopicData* topic, const Value& value, return true; } -void StorageImpl::NotifyValue(TopicData* topic, const Value& value, +void StorageImpl::NotifyValue(LocalTopic* topic, const Value& value, unsigned int eventFlags, bool isDuplicate, - const PublisherData* publisher) { + const LocalPublisher* publisher) { bool isNetwork = (eventFlags & NT_EVENT_VALUE_REMOTE) != 0; for (auto&& subscriber : topic->localSubscribers) { if (subscriber->active && @@ -1020,7 +1020,7 @@ void StorageImpl::NotifyValue(TopicData* topic, const Value& value, } } -void StorageImpl::PropertiesUpdated(TopicData* topic, const wpi::json& update, +void StorageImpl::PropertiesUpdated(LocalTopic* topic, const wpi::json& update, unsigned int eventFlags, bool sendNetwork, bool updateFlags) { DEBUG4("PropertiesUpdated({}, {}, {}, {}, {})", topic->name, update.dump(), @@ -1079,7 +1079,7 @@ void StorageImpl::PropertiesUpdated(TopicData* topic, const wpi::json& update, } } -void StorageImpl::RefreshPubSubActive(TopicData* topic, +void StorageImpl::RefreshPubSubActive(LocalTopic* topic, bool warnOnSubMismatch) { for (auto&& publisher : topic->localPublishers) { publisher->UpdateActive(); @@ -1096,9 +1096,9 @@ void StorageImpl::RefreshPubSubActive(TopicData* topic, } } -PublisherData* StorageImpl::AddLocalPublisher(TopicData* topic, - const wpi::json& properties, - const PubSubConfig& config) { +LocalPublisher* StorageImpl::AddLocalPublisher(LocalTopic* topic, + const wpi::json& properties, + const PubSubConfig& config) { bool didExist = topic->Exists(); auto publisher = m_publishers.Add(m_inst, topic, config); topic->localPublishers.Add(publisher); @@ -1144,8 +1144,8 @@ PublisherData* StorageImpl::AddLocalPublisher(TopicData* topic, return publisher; } -SubscriberData* StorageImpl::AddLocalSubscriber(TopicData* topic, - const PubSubConfig& config) { +LocalSubscriber* StorageImpl::AddLocalSubscriber(LocalTopic* topic, + const PubSubConfig& config) { DEBUG4("AddLocalSubscriber({})", topic->name); auto subscriber = m_subscribers.Add(m_inst, topic, config); topic->localSubscribers.Add(subscriber); @@ -1177,7 +1177,7 @@ SubscriberData* StorageImpl::AddLocalSubscriber(TopicData* topic, return subscriber; } -std::unique_ptr StorageImpl::RemoveLocalSubscriber( +std::unique_ptr StorageImpl::RemoveLocalSubscriber( NT_Subscriber subHandle) { auto subscriber = m_subscribers.Remove(subHandle); if (subscriber) { @@ -1195,7 +1195,7 @@ std::unique_ptr StorageImpl::RemoveLocalSubscriber( return subscriber; } -PublisherData* StorageImpl::PublishEntry(EntryData* entry, NT_Type type) { +LocalPublisher* StorageImpl::PublishEntry(LocalEntry* entry, NT_Type type) { if (entry->publisher) { return entry->publisher; } @@ -1222,7 +1222,7 @@ PublisherData* StorageImpl::PublishEntry(EntryData* entry, NT_Type type) { return entry->publisher; } -bool StorageImpl::PublishLocalValue(PublisherData* publisher, +bool StorageImpl::PublishLocalValue(LocalPublisher* publisher, const Value& value, bool force) { if (!value) { return false; diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.h b/ntcore/src/main/native/cpp/local/LocalStorageImpl.h index d82424701a0..fbdfdad8774 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.h +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.h @@ -47,13 +47,13 @@ class StorageImpl { // Network interface functions // - void NetworkAnnounce(TopicData* topic, std::string_view typeStr, + void NetworkAnnounce(LocalTopic* topic, std::string_view typeStr, const wpi::json& properties, std::optional pubuid); - void RemoveNetworkPublisher(TopicData* topic); - void NetworkPropertiesUpdate(TopicData* topic, const wpi::json& update, + void RemoveNetworkPublisher(LocalTopic* topic); + void NetworkPropertiesUpdate(LocalTopic* topic, const wpi::json& update, bool ack); - void ServerSetValue(TopicData* topic, const Value& value) { + void ServerSetValue(LocalTopic* topic, const Value& value) { if (SetValue(topic, value, NT_EVENT_VALUE_REMOTE, false, nullptr)) { if (topic->IsCached()) { topic->lastValueNetwork = value; @@ -69,9 +69,9 @@ class StorageImpl { // Topic functions // - TopicData* GetOrCreateTopic(std::string_view name); + LocalTopic* GetOrCreateTopic(std::string_view name); - template F> + template F> void ForEachTopic(std::string_view prefix, unsigned int types, F&& func) const { for (auto&& topic : m_topics) { @@ -88,7 +88,7 @@ class StorageImpl { } } - template F> + template F> void ForEachTopic(std::string_view prefix, std::span types, F&& func) const { for (auto&& topic : m_topics) { @@ -118,17 +118,17 @@ class StorageImpl { // Topic property functions // - void SetFlags(TopicData* topic, unsigned int flags); - void SetPersistent(TopicData* topic, bool value); - void SetRetained(TopicData* topic, bool value); - void SetCached(TopicData* topic, bool value); + void SetFlags(LocalTopic* topic, unsigned int flags); + void SetPersistent(LocalTopic* topic, bool value); + void SetRetained(LocalTopic* topic, bool value); + void SetCached(LocalTopic* topic, bool value); - void SetProperty(TopicData* topic, std::string_view name, + void SetProperty(LocalTopic* topic, std::string_view name, const wpi::json& value); - bool SetProperties(TopicData* topic, const wpi::json& update, + bool SetProperties(LocalTopic* topic, const wpi::json& update, bool sendNetwork); - void DeleteProperty(TopicData* topic, std::string_view name); + void DeleteProperty(LocalTopic* topic, std::string_view name); // // Value functions @@ -149,58 +149,58 @@ class StorageImpl { // Publish/Subscribe/Entry functions // - SubscriberData* Subscribe(TopicData* topic, NT_Type type, - std::string_view typeString, - const PubSubOptions& options); + LocalSubscriber* Subscribe(LocalTopic* topic, NT_Type type, + std::string_view typeString, + const PubSubOptions& options); - PublisherData* Publish(TopicData* topic, NT_Type type, - std::string_view typeStr, const wpi::json& properties, - const PubSubOptions& options); + LocalPublisher* Publish(LocalTopic* topic, NT_Type type, + std::string_view typeStr, const wpi::json& properties, + const PubSubOptions& options); - EntryData* GetEntry(TopicData* topicHandle, NT_Type type, - std::string_view typeStr, const PubSubOptions& options); - EntryData* GetEntry(std::string_view name); + LocalEntry* GetEntry(LocalTopic* topicHandle, NT_Type type, + std::string_view typeStr, const PubSubOptions& options); + LocalEntry* GetEntry(std::string_view name); void RemoveSubEntry(NT_Handle subentryHandle); - std::unique_ptr RemoveLocalPublisher(NT_Publisher pubHandle); + std::unique_ptr RemoveLocalPublisher(NT_Publisher pubHandle); // // Multi-subscriber functions // - MultiSubscriberData* AddMultiSubscriber( + LocalMultiSubscriber* AddMultiSubscriber( std::span prefixes, const PubSubOptions& options); - std::unique_ptr RemoveMultiSubscriber( + std::unique_ptr RemoveMultiSubscriber( NT_MultiSubscriber subHandle); // // Lookup functions // - TopicData* GetTopic(NT_Handle handle); - TopicData* GetTopicByHandle(NT_Topic topicHandle) { + LocalTopic* GetTopic(NT_Handle handle); + LocalTopic* GetTopicByHandle(NT_Topic topicHandle) { return m_topics.Get(topicHandle); } - TopicData* GetTopicByName(std::string_view name) { + LocalTopic* GetTopicByName(std::string_view name) { return m_nameTopics.lookup(name); } - TopicData* GetTopicById(int topicId) { + LocalTopic* GetTopicById(int topicId) { return m_topics.Get(Handle{m_inst, topicId, Handle::kTopic}); } - SubscriberData* GetSubEntry(NT_Handle subentryHandle); + LocalSubscriber* GetSubEntry(NT_Handle subentryHandle); - EntryData* GetEntryByHandle(NT_Entry entryHandle) { + LocalEntry* GetEntryByHandle(NT_Entry entryHandle) { return m_entries.Get(entryHandle); } - MultiSubscriberData* GetMultiSubscriberByHandle(NT_MultiSubscriber handle) { + LocalMultiSubscriber* GetMultiSubscriberByHandle(NT_MultiSubscriber handle) { return m_multiSubscribers.Get(handle); } - SubscriberData* GetSubscriberByHandle(NT_Subscriber handle) { + LocalSubscriber* GetSubscriberByHandle(NT_Subscriber handle) { return m_subscribers.Get(handle); } @@ -208,13 +208,13 @@ class StorageImpl { // Listener functions // - void AddListenerImpl(NT_Listener listenerHandle, TopicData* topic, + void AddListenerImpl(NT_Listener listenerHandle, LocalTopic* topic, unsigned int eventMask); - void AddListenerImpl(NT_Listener listenerHandle, SubscriberData* subscriber, + void AddListenerImpl(NT_Listener listenerHandle, LocalSubscriber* subscriber, unsigned int eventMask, NT_Handle subentryHandle, bool subscriberOwned); void AddListenerImpl(NT_Listener listenerHandle, - MultiSubscriberData* subscriber, unsigned int eventMask, + LocalMultiSubscriber* subscriber, unsigned int eventMask, bool subscriberOwned); void RemoveListener(NT_Listener listener, unsigned int mask); @@ -222,8 +222,8 @@ class StorageImpl { // Data log functions // - DataLoggerData* StartDataLog(wpi::log::DataLog& log, std::string_view prefix, - std::string_view logPrefix); + LocalDataLogger* StartDataLog(wpi::log::DataLog& log, std::string_view prefix, + std::string_view logPrefix); void StopDataLog(NT_DataLogger logger); // @@ -238,39 +238,39 @@ class StorageImpl { private: // topic functions - void NotifyTopic(TopicData* topic, unsigned int eventFlags); + void NotifyTopic(LocalTopic* topic, unsigned int eventFlags); - void CheckReset(TopicData* topic); + void CheckReset(LocalTopic* topic); - bool SetValue(TopicData* topic, const Value& value, unsigned int eventFlags, - bool suppressIfDuplicate, const PublisherData* publisher); - void NotifyValue(TopicData* topic, const Value& value, + bool SetValue(LocalTopic* topic, const Value& value, unsigned int eventFlags, + bool suppressIfDuplicate, const LocalPublisher* publisher); + void NotifyValue(LocalTopic* topic, const Value& value, unsigned int eventFlags, bool isDuplicate, - const PublisherData* publisher); + const LocalPublisher* publisher); - void PropertiesUpdated(TopicData* topic, const wpi::json& update, + void PropertiesUpdated(LocalTopic* topic, const wpi::json& update, unsigned int eventFlags, bool sendNetwork, bool updateFlags = true); - void RefreshPubSubActive(TopicData* topic, bool warnOnSubMismatch); + void RefreshPubSubActive(LocalTopic* topic, bool warnOnSubMismatch); - PublisherData* AddLocalPublisher(TopicData* topic, - const wpi::json& properties, - const PubSubConfig& options); + LocalPublisher* AddLocalPublisher(LocalTopic* topic, + const wpi::json& properties, + const PubSubConfig& options); - SubscriberData* AddLocalSubscriber(TopicData* topic, - const PubSubConfig& options); + LocalSubscriber* AddLocalSubscriber(LocalTopic* topic, + const PubSubConfig& options); - std::unique_ptr RemoveLocalSubscriber( + std::unique_ptr RemoveLocalSubscriber( NT_Subscriber subHandle); - EntryData* AddEntry(SubscriberData* subscriber) { + LocalEntry* AddEntry(LocalSubscriber* subscriber) { auto entry = m_entries.Add(m_inst, subscriber); subscriber->topic->entries.Add(entry); return entry; } - std::unique_ptr RemoveEntry(NT_Entry entryHandle) { + std::unique_ptr RemoveEntry(NT_Entry entryHandle) { auto entry = m_entries.Remove(entryHandle); if (entry) { entry->topic->entries.Remove(entry.get()); @@ -278,9 +278,9 @@ class StorageImpl { return entry; } - PublisherData* PublishEntry(EntryData* entry, NT_Type type); + LocalPublisher* PublishEntry(LocalEntry* entry, NT_Type type); - bool PublishLocalValue(PublisherData* publisher, const Value& value, + bool PublishLocalValue(LocalPublisher* publisher, const Value& value, bool force = false); private: @@ -290,21 +290,21 @@ class StorageImpl { net::ClientMessageHandler* m_network{nullptr}; // handle mappings - HandleMap m_topics; - HandleMap m_publishers; - HandleMap m_subscribers; - HandleMap m_entries; - HandleMap m_multiSubscribers; - HandleMap m_dataloggers; + HandleMap m_topics; + HandleMap m_publishers; + HandleMap m_subscribers; + HandleMap m_entries; + HandleMap m_multiSubscribers; + HandleMap m_dataloggers; // name mappings - wpi::StringMap m_nameTopics; + wpi::StringMap m_nameTopics; // listeners - wpi::DenseMap> m_listeners; + wpi::DenseMap> m_listeners; // string-based listeners - VectorSet m_topicPrefixListeners; + VectorSet m_topicPrefixListeners; // schema publishers wpi::StringMap m_schemas; diff --git a/ntcore/src/main/native/cpp/local/LocalSubscriber.h b/ntcore/src/main/native/cpp/local/LocalSubscriber.h index 5545185831f..cc3bf45e437 100644 --- a/ntcore/src/main/native/cpp/local/LocalSubscriber.h +++ b/ntcore/src/main/native/cpp/local/LocalSubscriber.h @@ -16,10 +16,10 @@ namespace nt::local { -struct SubscriberData { +struct LocalSubscriber { static constexpr auto kType = Handle::kSubscriber; - SubscriberData(NT_Subscriber handle, TopicData* topic, PubSubConfig config) + LocalSubscriber(NT_Subscriber handle, LocalTopic* topic, PubSubConfig config) : handle{handle}, topic{topic}, config{std::move(config)}, @@ -35,7 +35,7 @@ struct SubscriberData { // invariants wpi::SignalObject handle; - TopicData* topic; + LocalTopic* topic; PubSubConfig config; // whether or not the subscriber should actually receive values diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.h b/ntcore/src/main/native/cpp/local/LocalTopic.h index a3d5e2a3d14..1ab6dda264f 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.h +++ b/ntcore/src/main/native/cpp/local/LocalTopic.h @@ -18,19 +18,19 @@ namespace nt::local { -struct EntryData; -struct MultiSubscriberData; -struct PublisherData; -struct SubscriberData; +struct LocalEntry; +struct LocalMultiSubscriber; +struct LocalPublisher; +struct LocalSubscriber; constexpr bool IsSpecial(std::string_view name) { return name.empty() ? false : name.front() == '$'; } -struct TopicData { +struct LocalTopic { static constexpr auto kType = Handle::kTopic; - TopicData(NT_Topic handle, std::string_view name) + LocalTopic(NT_Topic handle, std::string_view name) : handle{handle}, name{name}, special{IsSpecial(name)} {} bool Exists() const { return onNetwork || !localPublishers.empty(); } @@ -59,18 +59,18 @@ struct TopicData { unsigned int flags{0}; // for NT3 APIs std::string propertiesStr{"{}"}; // cached string for GetTopicInfo() et al wpi::json properties = wpi::json::object(); - EntryData* entry{nullptr}; // cached entry for GetEntry() + LocalEntry* entry{nullptr}; // cached entry for GetEntry() bool onNetwork{false}; // true if there are any remote publishers bool lastValueFromNetwork{false}; - wpi::SmallVector datalogs; + wpi::SmallVector datalogs; NT_Type datalogType{NT_UNASSIGNED}; - VectorSet localPublishers; - VectorSet localSubscribers; - VectorSet multiSubscribers; - VectorSet entries; + VectorSet localPublishers; + VectorSet localSubscribers; + VectorSet multiSubscribers; + VectorSet entries; VectorSet listeners; }; From a215d898abebb29422d07c6144c1dea8f9be3b43 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Fri, 18 Oct 2024 23:37:20 -0700 Subject: [PATCH 19/26] [ntcore] LocalStorageImpl: Replace WPI_ERROR with ERR --- .../native/cpp/local/LocalStorageImpl.cpp | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp index 905ad302615..5c6d0ff4c48 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp @@ -387,9 +387,8 @@ LocalSubscriber* StorageImpl::Subscribe(LocalTopic* topic, NT_Type type, std::string_view typeStr, const PubSubOptions& options) { if (topic->localSubscribers.size() >= kMaxSubscribers) { - WPI_ERROR(m_logger, - "reached maximum number of subscribers to '{}', not subscribing", - topic->name); + ERR("reached maximum number of subscribers to '{}', not subscribing", + topic->name); return nullptr; } @@ -402,17 +401,14 @@ LocalPublisher* StorageImpl::Publish(LocalTopic* topic, NT_Type type, const wpi::json& properties, const PubSubOptions& options) { if (type == NT_UNASSIGNED || typeStr.empty()) { - WPI_ERROR( - m_logger, - "cannot publish '{}' with an unassigned type or empty type string", + ERR("cannot publish '{}' with an unassigned type or empty type string", topic->name); return nullptr; } if (topic->localPublishers.size() >= kMaxPublishers) { - WPI_ERROR(m_logger, - "reached maximum number of publishers to '{}', not publishing", - topic->name); + ERR("reached maximum number of publishers to '{}', not publishing", + topic->name); return nullptr; } @@ -424,9 +420,7 @@ LocalEntry* StorageImpl::GetEntry(LocalTopic* topic, NT_Type type, std::string_view typeStr, const PubSubOptions& options) { if (topic->localSubscribers.size() >= kMaxSubscribers) { - WPI_ERROR( - m_logger, - "reached maximum number of subscribers to '{}', not creating entry", + ERR("reached maximum number of subscribers to '{}', not creating entry", topic->name); return nullptr; } @@ -449,9 +443,7 @@ LocalEntry* StorageImpl::GetEntry(std::string_view name) { if (!topic->entry) { if (topic->localSubscribers.size() >= kMaxSubscribers) { - WPI_ERROR( - m_logger, - "reached maximum number of subscribers to '{}', not creating entry", + ERR("reached maximum number of subscribers to '{}', not creating entry", topic->name); return nullptr; } @@ -526,8 +518,7 @@ LocalMultiSubscriber* StorageImpl::AddMultiSubscriber( std::span prefixes, const PubSubOptions& options) { DEBUG4("AddMultiSubscriber({})", fmt::join(prefixes, ",")); if (m_multiSubscribers.size() >= kMaxMultiSubscribers) { - WPI_ERROR(m_logger, - "reached maximum number of multi-subscribers, not subscribing"); + ERR("reached maximum number of multi-subscribers, not subscribing"); return nullptr; } auto subscriber = m_multiSubscribers.Add(m_inst, prefixes, options); @@ -866,9 +857,8 @@ void StorageImpl::AddSchema(std::string_view name, std::string_view type, auto topic = GetOrCreateTopic(fullName); if (topic->localPublishers.size() >= kMaxPublishers) { - WPI_ERROR(m_logger, - "reached maximum number of publishers to '{}', not publishing", - topic->name); + ERR("reached maximum number of publishers to '{}', not publishing", + topic->name); return; } From 8fa4bcd5d3bc8ed845136178ba9c0182a3b1720e Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Sat, 19 Oct 2024 00:08:26 -0700 Subject: [PATCH 20/26] [ntcore] HandleMap: Use concepts for T --- ntcore/src/main/native/cpp/HandleMap.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ntcore/src/main/native/cpp/HandleMap.h b/ntcore/src/main/native/cpp/HandleMap.h index 03e73f9b15f..89b48668906 100644 --- a/ntcore/src/main/native/cpp/HandleMap.h +++ b/ntcore/src/main/native/cpp/HandleMap.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include @@ -13,8 +14,13 @@ namespace nt { +template +concept HandleType = requires { + { T::kType } -> std::convertible_to; +}; + // Utility wrapper class for our UidVectors -template +template class HandleMap : public wpi::UidVector, Size> { public: template From f8888c83a9eb9ba3227c78fb4e47fb2b3561a89e Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Sat, 19 Oct 2024 11:06:46 -0700 Subject: [PATCH 21/26] [ntcore] Refactor local topic DataLog --- .../native/cpp/local/LocalDataLoggerEntry.cpp | 1 - .../native/cpp/local/LocalDataLoggerEntry.h | 8 ++++ .../native/cpp/local/LocalStorageImpl.cpp | 33 ++--------------- .../src/main/native/cpp/local/LocalTopic.cpp | 37 +++++++++++++++++++ ntcore/src/main/native/cpp/local/LocalTopic.h | 9 +++++ 5 files changed, 58 insertions(+), 30 deletions(-) create mode 100644 ntcore/src/main/native/cpp/local/LocalTopic.cpp diff --git a/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp index ed5a1bba9df..938a5a4566e 100644 --- a/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp +++ b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.cpp @@ -5,7 +5,6 @@ #include "LocalDataLoggerEntry.h" #include -#include #include #include "networktables/NetworkTableValue.h" diff --git a/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h index 605635f9ab7..53ff480b19b 100644 --- a/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h +++ b/ntcore/src/main/native/cpp/local/LocalDataLoggerEntry.h @@ -4,9 +4,13 @@ #pragma once +#include + #include #include +#include + #include "ntcore_c.h" namespace wpi::log { @@ -28,6 +32,10 @@ struct LocalDataLoggerEntry { static std::string MakeMetadata(std::string_view properties); void Append(const Value& v); + void Finish(int64_t timestamp) { log->Finish(entry, timestamp); } + void SetMetadata(std::string_view metadata, int64_t timestamp) { + log->SetMetadata(entry, metadata, timestamp); + } wpi::log::DataLog* log; int entry; diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp index 5c6d0ff4c48..d9b0996df2f 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp @@ -823,13 +823,7 @@ void StorageImpl::StopDataLog(NT_DataLogger logger) { // finish any active entries auto now = Now(); for (auto&& topic : m_topics) { - auto it = - std::find_if(topic->datalogs.begin(), topic->datalogs.end(), - [&](const auto& elem) { return elem.logger == logger; }); - if (it != topic->datalogs.end()) { - it->log->Finish(it->entry, now); - topic->datalogs.erase(it); - } + topic->StartStopDataLog(datalogger.get(), now, false); } } } @@ -905,32 +899,13 @@ void StorageImpl::NotifyTopic(LocalTopic* topic, unsigned int eventFlags) { auto now = Now(); for (auto&& datalogger : m_dataloggers) { if (PrefixMatch(topic->name, datalogger->prefix, topic->special)) { - auto it = std::find_if(topic->datalogs.begin(), topic->datalogs.end(), - [&](const auto& elem) { - return elem.logger == datalogger->handle; - }); - if ((eventFlags & NT_EVENT_PUBLISH) != 0 && - it == topic->datalogs.end()) { - topic->datalogs.emplace_back(datalogger->log, - datalogger->Start(topic, now), - datalogger->handle); - topic->datalogType = topic->type; - } else if ((eventFlags & NT_EVENT_UNPUBLISH) != 0 && - it != topic->datalogs.end()) { - it->log->Finish(it->entry, now); - topic->datalogType = NT_UNASSIGNED; - topic->datalogs.erase(it); - } + topic->StartStopDataLog(datalogger.get(), now, + (eventFlags & NT_EVENT_PUBLISH) != 0); } } } } else if ((eventFlags & NT_EVENT_PROPERTIES) != 0) { - if (!topic->datalogs.empty()) { - auto metadata = LocalDataLoggerEntry::MakeMetadata(topic->propertiesStr); - for (auto&& datalog : topic->datalogs) { - datalog.log->SetMetadata(datalog.entry, metadata); - } - } + topic->UpdateDataLogProperties(); } } diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.cpp b/ntcore/src/main/native/cpp/local/LocalTopic.cpp new file mode 100644 index 00000000000..d788619cdfb --- /dev/null +++ b/ntcore/src/main/native/cpp/local/LocalTopic.cpp @@ -0,0 +1,37 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "LocalTopic.h" + +#include + +#include "local/LocalDataLogger.h" + +using namespace nt::local; + +void LocalTopic::StartStopDataLog(LocalDataLogger* logger, int64_t timestamp, + bool publish) { + auto it = std::find_if( + datalogs.begin(), datalogs.end(), + [&](const auto& elem) { return elem.logger == logger->handle; }); + if (publish && it == datalogs.end()) { + datalogs.emplace_back(logger->log, logger->Start(this, timestamp), + logger->handle); + datalogType = type; + } else if (!publish && it != datalogs.end()) { + it->Finish(timestamp); + datalogType = NT_UNASSIGNED; + datalogs.erase(it); + } +} + +void LocalTopic::UpdateDataLogProperties() { + if (!datalogs.empty()) { + auto now = Now(); + auto metadata = LocalDataLoggerEntry::MakeMetadata(propertiesStr); + for (auto&& datalog : datalogs) { + datalog.SetMetadata(metadata, now); + } + } +} diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.h b/ntcore/src/main/native/cpp/local/LocalTopic.h index 1ab6dda264f..433c1812490 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.h +++ b/ntcore/src/main/native/cpp/local/LocalTopic.h @@ -4,6 +4,8 @@ #pragma once +#include + #include #include @@ -13,11 +15,13 @@ #include "Handle.h" #include "VectorSet.h" +#include "local/LocalDataLogger.h" #include "local/LocalDataLoggerEntry.h" #include "ntcore_cpp.h" namespace nt::local { +struct LocalDataLogger; struct LocalEntry; struct LocalMultiSubscriber; struct LocalPublisher; @@ -37,6 +41,11 @@ struct LocalTopic { bool IsCached() const { return (flags & NT_UNCACHED) == 0; } + // starts if publish is true, stops if false + void StartStopDataLog(LocalDataLogger* logger, int64_t timestamp, + bool publish); + void UpdateDataLogProperties(); + TopicInfo GetTopicInfo() const { TopicInfo info; info.topic = handle; From 85ba06e97027459a5f76a82321fe1a25d616e02d Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Sat, 19 Oct 2024 11:51:20 -0700 Subject: [PATCH 22/26] [ntcore] Move properties and flags refresh to LocalTopic --- .../native/cpp/local/LocalStorageImpl.cpp | 51 ++----------------- .../src/main/native/cpp/local/LocalTopic.cpp | 46 +++++++++++++++++ ntcore/src/main/native/cpp/local/LocalTopic.h | 6 +++ 3 files changed, 57 insertions(+), 46 deletions(-) diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp index d9b0996df2f..a15abf74a9e 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp @@ -4,7 +4,6 @@ #include "LocalStorageImpl.h" -#include #include #include @@ -990,53 +989,13 @@ void StorageImpl::PropertiesUpdated(LocalTopic* topic, const wpi::json& update, bool updateFlags) { DEBUG4("PropertiesUpdated({}, {}, {}, {}, {})", topic->name, update.dump(), eventFlags, sendNetwork, updateFlags); - if (updateFlags) { - // set flags from properties - auto it = topic->properties.find("persistent"); - if (it != topic->properties.end()) { - if (auto val = it->get_ptr()) { - if (*val) { - topic->flags |= NT_PERSISTENT; - } else { - topic->flags &= ~NT_PERSISTENT; - } - } - } - it = topic->properties.find("retained"); - if (it != topic->properties.end()) { - if (auto val = it->get_ptr()) { - if (*val) { - topic->flags |= NT_RETAINED; - } else { - topic->flags &= ~NT_RETAINED; - } - } - } - it = topic->properties.find("cached"); - if (it != topic->properties.end()) { - if (auto val = it->get_ptr()) { - if (*val) { - topic->flags &= ~NT_UNCACHED; - } else { - topic->flags |= NT_UNCACHED; - } - } - } - - if ((topic->flags & NT_UNCACHED) != 0) { - topic->lastValue = {}; - topic->lastValueNetwork = {}; - topic->lastValueFromNetwork = false; - } - - if ((topic->flags & NT_UNCACHED) != 0 && - (topic->flags & NT_PERSISTENT) != 0) { - WARN("topic {}: disabling cached property disables persistent storage", - topic->name); - } + topic->RefreshProperties(updateFlags); + if (updateFlags && (topic->flags & NT_UNCACHED) != 0 && + (topic->flags & NT_PERSISTENT) != 0) { + WARN("topic {}: disabling cached property disables persistent storage", + topic->name); } - topic->propertiesStr = topic->properties.dump(); NotifyTopic(topic, eventFlags | NT_EVENT_PROPERTIES); // check local flag so we don't echo back received properties changes if (m_network && sendNetwork) { diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.cpp b/ntcore/src/main/native/cpp/local/LocalTopic.cpp index d788619cdfb..459024ee989 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.cpp +++ b/ntcore/src/main/native/cpp/local/LocalTopic.cpp @@ -35,3 +35,49 @@ void LocalTopic::UpdateDataLogProperties() { } } } + +void LocalTopic::RefreshProperties(bool updateFlags) { + if (updateFlags) { + RefreshFlags(); + } + propertiesStr = properties.dump(); +} + +void LocalTopic::RefreshFlags() { + auto it = properties.find("persistent"); + if (it != properties.end()) { + if (auto val = it->get_ptr()) { + if (*val) { + flags |= NT_PERSISTENT; + } else { + flags &= ~NT_PERSISTENT; + } + } + } + it = properties.find("retained"); + if (it != properties.end()) { + if (auto val = it->get_ptr()) { + if (*val) { + flags |= NT_RETAINED; + } else { + flags &= ~NT_RETAINED; + } + } + } + it = properties.find("cached"); + if (it != properties.end()) { + if (auto val = it->get_ptr()) { + if (*val) { + flags &= ~NT_UNCACHED; + } else { + flags |= NT_UNCACHED; + } + } + } + + if ((flags & NT_UNCACHED) != 0) { + lastValue = {}; + lastValueNetwork = {}; + lastValueFromNetwork = false; + } +} diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.h b/ntcore/src/main/native/cpp/local/LocalTopic.h index 433c1812490..824b01c2220 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.h +++ b/ntcore/src/main/native/cpp/local/LocalTopic.h @@ -46,6 +46,8 @@ struct LocalTopic { bool publish); void UpdateDataLogProperties(); + void RefreshProperties(bool updateFlags); + TopicInfo GetTopicInfo() const { TopicInfo info; info.topic = handle; @@ -81,6 +83,10 @@ struct LocalTopic { VectorSet multiSubscribers; VectorSet entries; VectorSet listeners; + + private: + // update flags from properties + void RefreshFlags(); }; } // namespace nt::local From 2285fc1fe288b44be08500b836b685462c795f15 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Sat, 19 Oct 2024 12:00:42 -0700 Subject: [PATCH 23/26] [ntcore] Further refactor of topic datalog --- .../main/native/cpp/local/LocalDataLogger.cpp | 15 ++++-------- .../main/native/cpp/local/LocalDataLogger.h | 3 ++- .../native/cpp/local/LocalStorageImpl.cpp | 23 +++---------------- .../main/native/cpp/local/LocalStorageImpl.h | 2 -- .../src/main/native/cpp/local/LocalTopic.cpp | 22 ++++++++++++++++-- ntcore/src/main/native/cpp/local/LocalTopic.h | 7 ++++-- 6 files changed, 35 insertions(+), 37 deletions(-) diff --git a/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp b/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp index 86028b2f37b..121c99ab39b 100644 --- a/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp +++ b/ntcore/src/main/native/cpp/local/LocalDataLogger.cpp @@ -8,22 +8,17 @@ #include #include -#include "local/LocalDataLoggerEntry.h" -#include "local/LocalTopic.h" - using namespace nt::local; -int LocalDataLogger::Start(LocalTopic* topic, int64_t time) { - std::string_view typeStr = topic->typeStr; +int LocalDataLogger::Start(std::string_view name, std::string_view typeStr, + std::string_view metadata, int64_t time) { // NT and DataLog use different standard representations for int and int[] if (typeStr == "int") { typeStr = "int64"; } else if (typeStr == "int[]") { typeStr = "int64[]"; } - return log.Start( - fmt::format( - "{}{}", logPrefix, - wpi::remove_prefix(topic->name, prefix).value_or(topic->name)), - typeStr, LocalDataLoggerEntry::MakeMetadata(topic->propertiesStr), time); + return log.Start(fmt::format("{}{}", logPrefix, + wpi::remove_prefix(name, prefix).value_or(name)), + typeStr, metadata, time); } diff --git a/ntcore/src/main/native/cpp/local/LocalDataLogger.h b/ntcore/src/main/native/cpp/local/LocalDataLogger.h index 66d32724300..c92e9ce337e 100644 --- a/ntcore/src/main/native/cpp/local/LocalDataLogger.h +++ b/ntcore/src/main/native/cpp/local/LocalDataLogger.h @@ -27,7 +27,8 @@ struct LocalDataLogger { std::string_view prefix, std::string_view logPrefix) : handle{handle}, log{log}, prefix{prefix}, logPrefix{logPrefix} {} - int Start(LocalTopic* topic, int64_t time); + int Start(std::string_view name, std::string_view typeStr, + std::string_view metadata, int64_t time); NT_DataLogger handle; wpi::log::DataLog& log; diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp index a15abf74a9e..c7d45780396 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp @@ -95,7 +95,7 @@ void StorageImpl::RemoveNetworkPublisher(LocalTopic* topic) { topic->onNetwork = false; if (didExist && !topic->Exists()) { DEBUG4("Unpublished {}", topic->name); - CheckReset(topic); + topic->ResetIfDoesNotExist(); NotifyTopic(topic, NT_EVENT_UNPUBLISH); } @@ -481,7 +481,7 @@ std::unique_ptr StorageImpl::RemoveLocalPublisher( bool didExist = topic->Exists(); topic->localPublishers.Remove(publisher.get()); if (didExist && !topic->Exists()) { - CheckReset(topic); + topic->ResetIfDoesNotExist(); NotifyTopic(topic, NT_EVENT_UNPUBLISH); } @@ -804,10 +804,7 @@ LocalDataLogger* StorageImpl::StartDataLog(wpi::log::DataLog& log, topic->type == NT_UNASSIGNED || topic->typeStr.empty()) { continue; } - topic->datalogs.emplace_back(log, datalogger->Start(topic.get(), now), - datalogger->handle); - topic->datalogType = topic->type; - + topic->StartStopDataLog(datalogger, now, true); // log current value, if any if (topic->lastValue) { topic->datalogs.back().Append(topic->lastValue); @@ -908,20 +905,6 @@ void StorageImpl::NotifyTopic(LocalTopic* topic, unsigned int eventFlags) { } } -void StorageImpl::CheckReset(LocalTopic* topic) { - if (topic->Exists()) { - return; - } - topic->lastValue = {}; - topic->lastValueNetwork = {}; - topic->lastValueFromNetwork = false; - topic->type = NT_UNASSIGNED; - topic->typeStr.clear(); - topic->flags = 0; - topic->properties = wpi::json::object(); - topic->propertiesStr = "{}"; -} - bool StorageImpl::SetValue(LocalTopic* topic, const Value& value, unsigned int eventFlags, bool suppressIfDuplicate, const LocalPublisher* publisher) { diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.h b/ntcore/src/main/native/cpp/local/LocalStorageImpl.h index fbdfdad8774..033dac5fd08 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.h +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.h @@ -240,8 +240,6 @@ class StorageImpl { // topic functions void NotifyTopic(LocalTopic* topic, unsigned int eventFlags); - void CheckReset(LocalTopic* topic); - bool SetValue(LocalTopic* topic, const Value& value, unsigned int eventFlags, bool suppressIfDuplicate, const LocalPublisher* publisher); void NotifyValue(LocalTopic* topic, const Value& value, diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.cpp b/ntcore/src/main/native/cpp/local/LocalTopic.cpp index 459024ee989..9543602c2a5 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.cpp +++ b/ntcore/src/main/native/cpp/local/LocalTopic.cpp @@ -16,8 +16,12 @@ void LocalTopic::StartStopDataLog(LocalDataLogger* logger, int64_t timestamp, datalogs.begin(), datalogs.end(), [&](const auto& elem) { return elem.logger == logger->handle; }); if (publish && it == datalogs.end()) { - datalogs.emplace_back(logger->log, logger->Start(this, timestamp), - logger->handle); + datalogs.emplace_back( + logger->log, + logger->Start(name, typeStr, + LocalDataLoggerEntry::MakeMetadata(propertiesStr), + timestamp), + logger->handle); datalogType = type; } else if (!publish && it != datalogs.end()) { it->Finish(timestamp); @@ -43,6 +47,20 @@ void LocalTopic::RefreshProperties(bool updateFlags) { propertiesStr = properties.dump(); } +void LocalTopic::ResetIfDoesNotExist() { + if (Exists()) { + return; + } + lastValue = {}; + lastValueNetwork = {}; + lastValueFromNetwork = false; + type = NT_UNASSIGNED; + typeStr.clear(); + flags = 0; + properties = wpi::json::object(); + propertiesStr = "{}"; +} + void LocalTopic::RefreshFlags() { auto it = properties.find("persistent"); if (it != properties.end()) { diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.h b/ntcore/src/main/native/cpp/local/LocalTopic.h index 824b01c2220..3b6301be51d 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.h +++ b/ntcore/src/main/native/cpp/local/LocalTopic.h @@ -58,6 +58,8 @@ struct LocalTopic { return info; } + void ResetIfDoesNotExist(); + // invariants wpi::SignalObject handle; std::string name; @@ -67,8 +69,7 @@ struct LocalTopic { Value lastValueNetwork; NT_Type type{NT_UNASSIGNED}; std::string typeStr; - unsigned int flags{0}; // for NT3 APIs - std::string propertiesStr{"{}"}; // cached string for GetTopicInfo() et al + unsigned int flags{0}; // for NT3 APIs wpi::json properties = wpi::json::object(); LocalEntry* entry{nullptr}; // cached entry for GetEntry() @@ -87,6 +88,8 @@ struct LocalTopic { private: // update flags from properties void RefreshFlags(); + + std::string propertiesStr{"{}"}; // cached string for GetTopicInfo() et al }; } // namespace nt::local From ba95c44296431d9cc32e9ecf52f1b42896a56f5a Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Sat, 19 Oct 2024 21:59:29 -0700 Subject: [PATCH 24/26] [ntcore] Local topic: Refactor flags --- ntcore/src/main/native/cpp/LocalStorage.h | 8 +- .../native/cpp/local/LocalStorageImpl.cpp | 98 ++------------ .../src/main/native/cpp/local/LocalTopic.cpp | 123 ++++++++++++++++-- ntcore/src/main/native/cpp/local/LocalTopic.h | 17 ++- 4 files changed, 148 insertions(+), 98 deletions(-) diff --git a/ntcore/src/main/native/cpp/LocalStorage.h b/ntcore/src/main/native/cpp/LocalStorage.h index 17f012cd598..9254756afc3 100644 --- a/ntcore/src/main/native/cpp/LocalStorage.h +++ b/ntcore/src/main/native/cpp/LocalStorage.h @@ -133,7 +133,7 @@ class LocalStorage final : public net::ILocalStorage { bool GetTopicPersistent(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { - return (topic->flags & NT_PERSISTENT) != 0; + return (topic->GetFlags() & NT_PERSISTENT) != 0; } else { return false; } @@ -149,7 +149,7 @@ class LocalStorage final : public net::ILocalStorage { bool GetTopicRetained(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { - return (topic->flags & NT_RETAINED) != 0; + return (topic->GetFlags() & NT_RETAINED) != 0; } else { return false; } @@ -165,7 +165,7 @@ class LocalStorage final : public net::ILocalStorage { bool GetTopicCached(NT_Topic topicHandle) { std::scoped_lock lock{m_mutex}; if (auto topic = m_impl.GetTopicByHandle(topicHandle)) { - return (topic->flags & NT_UNCACHED) == 0; + return (topic->GetFlags() & NT_UNCACHED) == 0; } else { return false; } @@ -388,7 +388,7 @@ class LocalStorage final : public net::ILocalStorage { unsigned int GetEntryFlags(NT_Entry entryHandle) { std::scoped_lock lock{m_mutex}; if (auto entry = m_impl.GetEntryByHandle(entryHandle)) { - return entry->subscriber->topic->flags; + return entry->subscriber->topic->GetFlags(); } else { return 0; } diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp index c7d45780396..adc0060d554 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp @@ -195,119 +195,48 @@ LocalTopic* StorageImpl::GetOrCreateTopic(std::string_view name) { // void StorageImpl::SetFlags(LocalTopic* topic, unsigned int flags) { - wpi::json update = wpi::json::object(); - if ((flags & NT_PERSISTENT) != 0) { - topic->properties["persistent"] = true; - update["persistent"] = true; - } else { - topic->properties.erase("persistent"); - update["persistent"] = wpi::json(); - } - if ((flags & NT_RETAINED) != 0) { - topic->properties["retained"] = true; - update["retained"] = true; - } else { - topic->properties.erase("retained"); - update["retained"] = wpi::json(); - } - if ((flags & NT_UNCACHED) != 0) { - topic->properties["cached"] = false; - update["cached"] = false; - } else { - topic->properties.erase("cached"); - update["cached"] = wpi::json(); - } - if ((flags & NT_UNCACHED) != 0) { - topic->lastValue = {}; - topic->lastValueNetwork = {}; - topic->lastValueFromNetwork = false; - } + wpi::json update = topic->SetFlags(flags); if ((flags & NT_UNCACHED) != 0 && (flags & NT_PERSISTENT) != 0) { WARN("topic {}: disabling cached property disables persistent storage", topic->name); } - topic->flags = flags; if (!update.empty()) { PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); } } void StorageImpl::SetPersistent(LocalTopic* topic, bool value) { - wpi::json update = wpi::json::object(); - if (value) { - topic->flags |= NT_PERSISTENT; - topic->properties["persistent"] = true; - update["persistent"] = true; - } else { - topic->flags &= ~NT_PERSISTENT; - topic->properties.erase("persistent"); - update["persistent"] = wpi::json(); - } - PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); + PropertiesUpdated(topic, topic->SetPersistent(value), NT_EVENT_NONE, true, + false); } void StorageImpl::SetRetained(LocalTopic* topic, bool value) { - wpi::json update = wpi::json::object(); - if (value) { - topic->flags |= NT_RETAINED; - topic->properties["retained"] = true; - update["retained"] = true; - } else { - topic->flags &= ~NT_RETAINED; - topic->properties.erase("retained"); - update["retained"] = wpi::json(); - } - PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); + PropertiesUpdated(topic, topic->SetRetained(value), NT_EVENT_NONE, true, + false); } void StorageImpl::SetCached(LocalTopic* topic, bool value) { - wpi::json update = wpi::json::object(); - if (value) { - topic->flags &= ~NT_UNCACHED; - topic->properties.erase("cached"); - update["cached"] = wpi::json(); - } else { - topic->flags |= NT_UNCACHED; - topic->properties["cached"] = false; - update["cached"] = false; - } - PropertiesUpdated(topic, update, NT_EVENT_NONE, true, false); + PropertiesUpdated(topic, topic->SetCached(value), NT_EVENT_NONE, true, false); } void StorageImpl::SetProperty(LocalTopic* topic, std::string_view name, const wpi::json& value) { - if (value.is_null()) { - topic->properties.erase(name); - } else { - topic->properties[name] = value; - } - wpi::json update = wpi::json::object(); - update[name] = value; - PropertiesUpdated(topic, update, NT_EVENT_NONE, true); + PropertiesUpdated(topic, topic->SetProperty(name, value), NT_EVENT_NONE, + true); } bool StorageImpl::SetProperties(LocalTopic* topic, const wpi::json& update, bool sendNetwork) { - if (!update.is_object()) { - return false; - } DEBUG4("SetProperties({},{})", topic->name, sendNetwork); - for (auto&& change : update.items()) { - if (change.value().is_null()) { - topic->properties.erase(change.key()); - } else { - topic->properties[change.key()] = change.value(); - } + if (!topic->SetProperties(update)) { + return false; } PropertiesUpdated(topic, update, NT_EVENT_NONE, sendNetwork); return true; } void StorageImpl::DeleteProperty(LocalTopic* topic, std::string_view name) { - topic->properties.erase(name); - wpi::json update = wpi::json::object(); - update[name] = wpi::json(); - PropertiesUpdated(topic, update, NT_EVENT_NONE, true); + PropertiesUpdated(topic, topic->DeleteProperty(name), NT_EVENT_NONE, true); } // @@ -973,8 +902,9 @@ void StorageImpl::PropertiesUpdated(LocalTopic* topic, const wpi::json& update, DEBUG4("PropertiesUpdated({}, {}, {}, {}, {})", topic->name, update.dump(), eventFlags, sendNetwork, updateFlags); topic->RefreshProperties(updateFlags); - if (updateFlags && (topic->flags & NT_UNCACHED) != 0 && - (topic->flags & NT_PERSISTENT) != 0) { + unsigned int flags = topic->GetFlags(); + if (updateFlags && (flags & NT_UNCACHED) != 0 && + (flags & NT_PERSISTENT) != 0) { WARN("topic {}: disabling cached property disables persistent storage", topic->name); } diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.cpp b/ntcore/src/main/native/cpp/local/LocalTopic.cpp index 9543602c2a5..f4a53195429 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.cpp +++ b/ntcore/src/main/native/cpp/local/LocalTopic.cpp @@ -40,6 +40,113 @@ void LocalTopic::UpdateDataLogProperties() { } } +wpi::json LocalTopic::SetFlags(unsigned int flags) { + wpi::json update = wpi::json::object(); + if ((flags & NT_PERSISTENT) != 0) { + properties["persistent"] = true; + update["persistent"] = true; + } else { + properties.erase("persistent"); + update["persistent"] = wpi::json(); + } + if ((flags & NT_RETAINED) != 0) { + properties["retained"] = true; + update["retained"] = true; + } else { + properties.erase("retained"); + update["retained"] = wpi::json(); + } + if ((flags & NT_UNCACHED) != 0) { + properties["cached"] = false; + update["cached"] = false; + } else { + properties.erase("cached"); + update["cached"] = wpi::json(); + } + if ((flags & NT_UNCACHED) != 0) { + lastValue = {}; + lastValueNetwork = {}; + lastValueFromNetwork = false; + } + this->m_flags = flags; + return update; +} + +wpi::json LocalTopic::SetPersistent(bool value) { + wpi::json update = wpi::json::object(); + if (value) { + m_flags |= NT_PERSISTENT; + properties["persistent"] = true; + update["persistent"] = true; + } else { + m_flags &= ~NT_PERSISTENT; + properties.erase("persistent"); + update["persistent"] = wpi::json(); + } + return update; +} + +wpi::json LocalTopic::SetRetained(bool value) { + wpi::json update = wpi::json::object(); + if (value) { + m_flags |= NT_RETAINED; + properties["retained"] = true; + update["retained"] = true; + } else { + m_flags &= ~NT_RETAINED; + properties.erase("retained"); + update["retained"] = wpi::json(); + } + return update; +} + +wpi::json LocalTopic::SetCached(bool value) { + wpi::json update = wpi::json::object(); + if (value) { + m_flags &= ~NT_UNCACHED; + properties.erase("cached"); + update["cached"] = wpi::json(); + } else { + m_flags |= NT_UNCACHED; + properties["cached"] = false; + update["cached"] = false; + } + return update; +} + +wpi::json LocalTopic::SetProperty(std::string_view name, + const wpi::json& value) { + if (value.is_null()) { + properties.erase(name); + } else { + properties[name] = value; + } + wpi::json update = wpi::json::object(); + update[name] = value; + return update; +} + +wpi::json LocalTopic::DeleteProperty(std::string_view name) { + properties.erase(name); + wpi::json update = wpi::json::object(); + update[name] = wpi::json(); + return update; +} + +bool LocalTopic::SetProperties(const wpi::json& update) { + if (!update.is_object()) { + return false; + } + for (auto&& change : update.items()) { + if (change.value().is_null()) { + properties.erase(change.key()); + } else { + properties[change.key()] = change.value(); + } + } + return true; +} + void LocalTopic::RefreshProperties(bool updateFlags) { if (updateFlags) { RefreshFlags(); @@ -56,7 +163,7 @@ void LocalTopic::ResetIfDoesNotExist() { lastValueFromNetwork = false; type = NT_UNASSIGNED; typeStr.clear(); - flags = 0; + m_flags = 0; properties = wpi::json::object(); propertiesStr = "{}"; } @@ -66,9 +173,9 @@ void LocalTopic::RefreshFlags() { if (it != properties.end()) { if (auto val = it->get_ptr()) { if (*val) { - flags |= NT_PERSISTENT; + m_flags |= NT_PERSISTENT; } else { - flags &= ~NT_PERSISTENT; + m_flags &= ~NT_PERSISTENT; } } } @@ -76,9 +183,9 @@ void LocalTopic::RefreshFlags() { if (it != properties.end()) { if (auto val = it->get_ptr()) { if (*val) { - flags |= NT_RETAINED; + m_flags |= NT_RETAINED; } else { - flags &= ~NT_RETAINED; + m_flags &= ~NT_RETAINED; } } } @@ -86,14 +193,14 @@ void LocalTopic::RefreshFlags() { if (it != properties.end()) { if (auto val = it->get_ptr()) { if (*val) { - flags &= ~NT_UNCACHED; + m_flags &= ~NT_UNCACHED; } else { - flags |= NT_UNCACHED; + m_flags |= NT_UNCACHED; } } } - if ((flags & NT_UNCACHED) != 0) { + if ((m_flags & NT_UNCACHED) != 0) { lastValue = {}; lastValueNetwork = {}; lastValueFromNetwork = false; diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.h b/ntcore/src/main/native/cpp/local/LocalTopic.h index 3b6301be51d..cc6304e1ed5 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.h +++ b/ntcore/src/main/native/cpp/local/LocalTopic.h @@ -39,13 +39,26 @@ struct LocalTopic { bool Exists() const { return onNetwork || !localPublishers.empty(); } - bool IsCached() const { return (flags & NT_UNCACHED) == 0; } + bool IsCached() const { return (m_flags & NT_UNCACHED) == 0; } // starts if publish is true, stops if false void StartStopDataLog(LocalDataLogger* logger, int64_t timestamp, bool publish); void UpdateDataLogProperties(); + unsigned int GetFlags() const { return m_flags; } + + // these return update json + wpi::json SetFlags(unsigned int flags); + wpi::json SetPersistent(bool value); + wpi::json SetRetained(bool value); + wpi::json SetCached(bool value); + wpi::json SetProperty(std::string_view name, const wpi::json& value); + wpi::json DeleteProperty(std::string_view name); + + // returns false if not object + bool SetProperties(const wpi::json& update); + void RefreshProperties(bool updateFlags); TopicInfo GetTopicInfo() const { @@ -69,7 +82,6 @@ struct LocalTopic { Value lastValueNetwork; NT_Type type{NT_UNASSIGNED}; std::string typeStr; - unsigned int flags{0}; // for NT3 APIs wpi::json properties = wpi::json::object(); LocalEntry* entry{nullptr}; // cached entry for GetEntry() @@ -89,6 +101,7 @@ struct LocalTopic { // update flags from properties void RefreshFlags(); + unsigned int m_flags{0}; // for NT3 APIs std::string propertiesStr{"{}"}; // cached string for GetTopicInfo() et al }; From 9a6ef9fa4a5b1432c8aa6a7c67785ceab81b12cd Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Sat, 19 Oct 2024 22:07:27 -0700 Subject: [PATCH 25/26] [ntcore] Local topic: A bit more properties refactor --- .../native/cpp/local/LocalStorageImpl.cpp | 15 +---------- .../src/main/native/cpp/local/LocalTopic.cpp | 26 ++++++++++++++++--- ntcore/src/main/native/cpp/local/LocalTopic.h | 7 +++-- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp index adc0060d554..4f11ab3796a 100644 --- a/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp +++ b/ntcore/src/main/native/cpp/local/LocalStorageImpl.cpp @@ -65,20 +65,7 @@ void StorageImpl::NetworkAnnounce(LocalTopic* topic, std::string_view typeStr, // may be properties update, but need to compare to see if it actually // changed to determine whether to update string / send event - wpi::json update = wpi::json::object(); - // added/changed - for (auto&& prop : properties.items()) { - auto it = topic->properties.find(prop.key()); - if (it == topic->properties.end() || *it != prop.value()) { - update[prop.key()] = prop.value(); - } - } - // removed - for (auto&& prop : topic->properties.items()) { - if (properties.find(prop.key()) == properties.end()) { - update[prop.key()] = wpi::json(); - } - } + wpi::json update = topic->CompareProperties(properties); if (!update.empty()) { topic->properties = properties; PropertiesUpdated(topic, update, event, false); diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.cpp b/ntcore/src/main/native/cpp/local/LocalTopic.cpp index f4a53195429..2803dd5a31f 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.cpp +++ b/ntcore/src/main/native/cpp/local/LocalTopic.cpp @@ -19,7 +19,7 @@ void LocalTopic::StartStopDataLog(LocalDataLogger* logger, int64_t timestamp, datalogs.emplace_back( logger->log, logger->Start(name, typeStr, - LocalDataLoggerEntry::MakeMetadata(propertiesStr), + LocalDataLoggerEntry::MakeMetadata(m_propertiesStr), timestamp), logger->handle); datalogType = type; @@ -33,7 +33,7 @@ void LocalTopic::StartStopDataLog(LocalDataLogger* logger, int64_t timestamp, void LocalTopic::UpdateDataLogProperties() { if (!datalogs.empty()) { auto now = Now(); - auto metadata = LocalDataLoggerEntry::MakeMetadata(propertiesStr); + auto metadata = LocalDataLoggerEntry::MakeMetadata(m_propertiesStr); for (auto&& datalog : datalogs) { datalog.SetMetadata(metadata, now); } @@ -151,7 +151,25 @@ void LocalTopic::RefreshProperties(bool updateFlags) { if (updateFlags) { RefreshFlags(); } - propertiesStr = properties.dump(); + m_propertiesStr = properties.dump(); +} + +wpi::json LocalTopic::CompareProperties(const wpi::json props) { + wpi::json update = wpi::json::object(); + // added/changed + for (auto&& prop : props.items()) { + auto it = properties.find(prop.key()); + if (it == properties.end() || *it != prop.value()) { + update[prop.key()] = prop.value(); + } + } + // removed + for (auto&& prop : properties.items()) { + if (props.find(prop.key()) == props.end()) { + update[prop.key()] = wpi::json(); + } + } + return update; } void LocalTopic::ResetIfDoesNotExist() { @@ -165,7 +183,7 @@ void LocalTopic::ResetIfDoesNotExist() { typeStr.clear(); m_flags = 0; properties = wpi::json::object(); - propertiesStr = "{}"; + m_propertiesStr = "{}"; } void LocalTopic::RefreshFlags() { diff --git a/ntcore/src/main/native/cpp/local/LocalTopic.h b/ntcore/src/main/native/cpp/local/LocalTopic.h index cc6304e1ed5..028aeb02c9f 100644 --- a/ntcore/src/main/native/cpp/local/LocalTopic.h +++ b/ntcore/src/main/native/cpp/local/LocalTopic.h @@ -61,13 +61,16 @@ struct LocalTopic { void RefreshProperties(bool updateFlags); + // returns update json + wpi::json CompareProperties(const wpi::json props); + TopicInfo GetTopicInfo() const { TopicInfo info; info.topic = handle; info.name = name; info.type = type; info.type_str = typeStr; - info.properties = propertiesStr; + info.properties = m_propertiesStr; return info; } @@ -102,7 +105,7 @@ struct LocalTopic { void RefreshFlags(); unsigned int m_flags{0}; // for NT3 APIs - std::string propertiesStr{"{}"}; // cached string for GetTopicInfo() et al + std::string m_propertiesStr{"{}"}; // cached string for GetTopicInfo() et al }; } // namespace nt::local From 207df786a7eee976518b2aee372590071f32a009 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Mon, 21 Oct 2024 19:14:32 -0700 Subject: [PATCH 26/26] [ntcore] Use Endian.h in WireEncoder3 --- .../src/main/native/cpp/net3/WireEncoder3.cpp | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/ntcore/src/main/native/cpp/net3/WireEncoder3.cpp b/ntcore/src/main/native/cpp/net3/WireEncoder3.cpp index d1e4c784358..7f587c9f604 100644 --- a/ntcore/src/main/native/cpp/net3/WireEncoder3.cpp +++ b/ntcore/src/main/native/cpp/net3/WireEncoder3.cpp @@ -4,6 +4,7 @@ #include "WireEncoder3.h" +#include #include #include #include @@ -19,28 +20,21 @@ static void Write8(wpi::raw_ostream& os, uint8_t val) { } static void Write16(wpi::raw_ostream& os, uint16_t val) { - os << std::span{{static_cast((val >> 8) & 0xff), - static_cast(val & 0xff)}}; + uint8_t buf[2]; + wpi::support::endian::write16be(buf, val); + os << buf; } static void Write32(wpi::raw_ostream& os, uint32_t val) { - os << std::span{{static_cast((val >> 24) & 0xff), - static_cast((val >> 16) & 0xff), - static_cast((val >> 8) & 0xff), - static_cast(val & 0xff)}}; + uint8_t buf[4]; + wpi::support::endian::write32be(buf, val); + os << buf; } static void WriteDouble(wpi::raw_ostream& os, double val) { - // The highest performance way to do this, albeit non-portable. - uint64_t v = wpi::bit_cast(val); - os << std::span{{static_cast((v >> 56) & 0xff), - static_cast((v >> 48) & 0xff), - static_cast((v >> 40) & 0xff), - static_cast((v >> 32) & 0xff), - static_cast((v >> 24) & 0xff), - static_cast((v >> 16) & 0xff), - static_cast((v >> 8) & 0xff), - static_cast(v & 0xff)}}; + uint8_t buf[8]; + wpi::support::endian::write64be(buf, wpi::bit_cast(val)); + os << buf; } static void WriteString(wpi::raw_ostream& os, std::string_view str) {