From 4b989e6113262ed29b5d874b474e9bd1ada68516 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sat, 27 Jul 2024 19:50:51 +0800 Subject: [PATCH 01/13] fix: Resolve the inconsistency between member configuration and log index in snapshot meta --- cmake/braft.cmake | 4 ++-- pikiwidb.conf | 2 +- save_load.sh | 2 +- src/praft/praft.cc | 13 ++++++++++++- src/praft/praft.h | 3 +++ src/praft/psnapshot.cc | 21 ++++++++++++++++++++- 6 files changed, 39 insertions(+), 6 deletions(-) mode change 100755 => 100644 save_load.sh diff --git a/cmake/braft.cmake b/cmake/braft.cmake index bb8e6882..2cf294c8 100644 --- a/cmake/braft.cmake +++ b/cmake/braft.cmake @@ -19,8 +19,8 @@ ExternalProject_Add( ${EXTERNAL_PROJECT_LOG_ARGS} DEPENDS brpc # The pr on braft is not merged, so I am using my own warehouse to run the test for the time being - GIT_REPOSITORY "https://github.com/pikiwidb/braft.git" - GIT_TAG v1.1.2-alpha2 + GIT_REPOSITORY https://github.com/panlei-coder/braft.git # "https://github.com/pikiwidb/braft.git" + GIT_TAG get_configuration # v1.1.2-alpha2 GIT_SHALLOW true CMAKE_ARGS -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} diff --git a/pikiwidb.conf b/pikiwidb.conf index 38d7371c..775bb857 100644 --- a/pikiwidb.conf +++ b/pikiwidb.conf @@ -343,6 +343,6 @@ rocksdb-ttl-second 604800 rocksdb-periodic-second 259200; ############################### RAFT ############################### -use-raft no +use-raft no # Braft relies on brpc to communicate via the default port number plus the port offset raft-port-offset 10 diff --git a/save_load.sh b/save_load.sh old mode 100755 new mode 100644 index 09f36c26..435dc48e --- a/save_load.sh +++ b/save_load.sh @@ -15,4 +15,4 @@ redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset redis-cli -p 7777 raft.node dosnapshot redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset -redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 +redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 \ No newline at end of file diff --git a/src/praft/praft.cc b/src/praft/praft.cc index c171ff4b..92486d1f 100644 --- a/src/praft/praft.cc +++ b/src/praft/praft.cc @@ -259,6 +259,16 @@ storage::LogIndex PRaft::GetLastLogIndex(bool is_flush) { return node_->get_last_log_index(is_flush); } +void PRaft::GetConfigurationByIndex(const int64_t index, braft::ConfigurationEntry* conf, + braft::ConfigurationEntry* learner_conf) { + if (!node_) { + ERROR("Node is not initialized"); + return; + } + + node_->get_configuration(index, conf, learner_conf); +} + void PRaft::SendNodeRequest(PClient* client) { assert(client); @@ -663,8 +673,9 @@ int PRaft::on_snapshot_load(braft::SnapshotReader* reader) { 2. When a node is improperly shut down and restarted, the minimum flush-index should be obtained as the starting point for fault recovery. */ + // replay from uint64_t replay_point = PSTORE.GetBackend(db_id_)->GetStorage()->GetSmallestFlushedLogIndex(); - node_->set_self_playback_point(replay_point); + node_->set_last_applied_index_and_term(replay_point); is_node_first_start_up_ = false; INFO("set replay_point: {}", replay_point); diff --git a/src/praft/praft.h b/src/praft/praft.h index 60c9e475..c09e1dca 100644 --- a/src/praft/praft.h +++ b/src/praft/praft.h @@ -14,6 +14,7 @@ #include #include +#include "braft/configuration_manager.h" #include "braft/file_system_adaptor.h" #include "braft/raft.h" #include "brpc/server.h" @@ -140,6 +141,8 @@ class PRaft : public braft::StateMachine { butil::Status GetListPeers(std::vector* peers); storage::LogIndex GetTerm(uint64_t log_index); storage::LogIndex GetLastLogIndex(bool is_flush = false); + void GetConfigurationByIndex(const int64_t index, braft::ConfigurationEntry* conf, + braft::ConfigurationEntry* learner_conf); bool IsInitialized() const { return node_ != nullptr && server_ != nullptr; } diff --git a/src/praft/psnapshot.cc b/src/praft/psnapshot.cc index 28fc99c0..1dc4e1f7 100644 --- a/src/praft/psnapshot.cc +++ b/src/praft/psnapshot.cc @@ -10,6 +10,8 @@ #include "psnapshot.h" +#include "braft/configuration.h" +#include "braft/configuration_manager.h" #include "braft/local_file_meta.pb.h" #include "braft/snapshot.h" #include "butil/files/file_path.h" @@ -79,12 +81,29 @@ braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int o PSTORE.HandleTaskSpecificDB(tasks); AddAllFiles(snapshot_path, &snapshot_meta_memtable, snapshot_path); - // update snapshot last log index and last_log_term + // update snapshot last log index, last_log_term, conf of last log index and learners auto& new_meta = const_cast(snapshot_meta_memtable.meta()); auto last_log_index = PSTORE.GetBackend(db_id)->GetStorage()->GetSmallestFlushedLogIndex(); new_meta.set_last_included_index(last_log_index); auto last_log_term = PRAFT.GetTerm(last_log_index); new_meta.set_last_included_term(last_log_term); + braft::ConfigurationEntry conf_entry; + braft::ConfigurationEntry learner_conf_entry; + PRAFT.GetConfigurationByIndex(last_log_index, &conf_entry, &learner_conf_entry); + new_meta.clear_peers(); + for (braft::Configuration::const_iterator iter = conf_entry.conf.begin(); iter != conf_entry.conf.end(); ++iter) { + *new_meta.add_peers() = iter->to_string(); + } + new_meta.clear_old_peers(); + for (braft::Configuration::const_iterator iter = conf_entry.old_conf.begin(); iter != conf_entry.old_conf.end(); + ++iter) { + *new_meta.add_old_peers() = iter->to_string(); + } + new_meta.clear_learners(); + for (braft::Configuration::const_iterator iter = learner_conf_entry.conf.begin(); + iter != learner_conf_entry.conf.end(); ++iter) { + *new_meta.add_learners() = iter->to_string(); + } INFO("Succeed to fix db_{} snapshot meta: {}, {}", db_id, last_log_index, last_log_term); auto rc = snapshot_meta_memtable.save_to_file(fs, meta_path); From 6af2c77494f208b64462c563b78d83e15f35bca1 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 28 Jul 2024 01:49:06 +0800 Subject: [PATCH 02/13] fix: fix a bug --- build.sh | 2 +- src/pikiwidb.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 644a5df6..53532ea2 100755 --- a/build.sh +++ b/build.sh @@ -50,7 +50,7 @@ do --) shift break - ;; + ;; esac shift done diff --git a/src/pikiwidb.cc b/src/pikiwidb.cc index 5108e8d9..3bf60c5e 100644 --- a/src/pikiwidb.cc +++ b/src/pikiwidb.cc @@ -227,7 +227,7 @@ static int InitLimit() { limit.rlim_cur = maxfiles; limit.rlim_max = maxfiles; if (setrlimit(RLIMIT_NOFILE, &limit) != -1) { - WARN("your 'limit -n ' of {} is not enough for PikiwiDB to start. PikiwiDB have successfully reconfig it to ", + WARN("your 'limit -n ' of {} is not enough for PikiwiDB to start. PikiwiDB have successfully reconfig it to {}", old_limit, limit.rlim_cur); } else { ERROR( @@ -282,7 +282,7 @@ int main(int ac, char* av[]) { daemonize(); } - InitLimit(); + // InitLimit(); // Code problem, program startup direct segment error pstd::InitRandom(); SignalSetup(); InitLogs(); From 08f323e4a0d35b8df0d9465796a5be41acc80637 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 28 Jul 2024 02:18:08 +0800 Subject: [PATCH 03/13] fix: fix the bug of conf file --- pikiwidb.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pikiwidb.conf b/pikiwidb.conf index 775bb857..e72a5e71 100644 --- a/pikiwidb.conf +++ b/pikiwidb.conf @@ -343,6 +343,6 @@ rocksdb-ttl-second 604800 rocksdb-periodic-second 259200; ############################### RAFT ############################### -use-raft no +use-raft no # Braft relies on brpc to communicate via the default port number plus the port offset -raft-port-offset 10 +raft-port-offset 10 \ No newline at end of file From bae5da681bc56f7d8b976ee5c43ed74cd42fc3ca Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 28 Jul 2024 08:43:20 +0800 Subject: [PATCH 04/13] fix: fix a comment --- cmake/braft.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/braft.cmake b/cmake/braft.cmake index 2cf294c8..79bf4e89 100644 --- a/cmake/braft.cmake +++ b/cmake/braft.cmake @@ -19,8 +19,8 @@ ExternalProject_Add( ${EXTERNAL_PROJECT_LOG_ARGS} DEPENDS brpc # The pr on braft is not merged, so I am using my own warehouse to run the test for the time being - GIT_REPOSITORY https://github.com/panlei-coder/braft.git # "https://github.com/pikiwidb/braft.git" - GIT_TAG get_configuration # v1.1.2-alpha2 + GIT_REPOSITORY "https://github.com/pikiwidb/braft.git" + GIT_TAG v1.1.2.1 GIT_SHALLOW true CMAKE_ARGS -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} From c074a99dbc2fa50c9df8b2c6633774696add1d66 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 28 Jul 2024 08:51:58 +0800 Subject: [PATCH 05/13] fix: Add a blank line at the end of the pikiwidb.conf file --- pikiwidb.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pikiwidb.conf b/pikiwidb.conf index e72a5e71..38d7371c 100644 --- a/pikiwidb.conf +++ b/pikiwidb.conf @@ -345,4 +345,4 @@ rocksdb-periodic-second 259200; ############################### RAFT ############################### use-raft no # Braft relies on brpc to communicate via the default port number plus the port offset -raft-port-offset 10 \ No newline at end of file +raft-port-offset 10 From 46eb97809bb97be98a318d089b12534961a0f5ab Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 25 Aug 2024 10:45:21 +0800 Subject: [PATCH 06/13] fix: fix comment --- src/praft/psnapshot.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/praft/psnapshot.cc b/src/praft/psnapshot.cc index 1dc4e1f7..cc2b9de6 100644 --- a/src/praft/psnapshot.cc +++ b/src/praft/psnapshot.cc @@ -91,17 +91,15 @@ braft::FileAdaptor* PPosixFileSystemAdaptor::open(const std::string& path, int o braft::ConfigurationEntry learner_conf_entry; PRAFT.GetConfigurationByIndex(last_log_index, &conf_entry, &learner_conf_entry); new_meta.clear_peers(); - for (braft::Configuration::const_iterator iter = conf_entry.conf.begin(); iter != conf_entry.conf.end(); ++iter) { + for (auto iter = conf_entry.conf.begin(); iter != conf_entry.conf.end(); ++iter) { *new_meta.add_peers() = iter->to_string(); } new_meta.clear_old_peers(); - for (braft::Configuration::const_iterator iter = conf_entry.old_conf.begin(); iter != conf_entry.old_conf.end(); - ++iter) { + for (auto iter = conf_entry.old_conf.begin(); iter != conf_entry.old_conf.end(); ++iter) { *new_meta.add_old_peers() = iter->to_string(); } new_meta.clear_learners(); - for (braft::Configuration::const_iterator iter = learner_conf_entry.conf.begin(); - iter != learner_conf_entry.conf.end(); ++iter) { + for (auto iter = learner_conf_entry.conf.begin(); iter != learner_conf_entry.conf.end(); ++iter) { *new_meta.add_learners() = iter->to_string(); } INFO("Succeed to fix db_{} snapshot meta: {}, {}", db_id, last_log_index, last_log_term); From df2344758804da58b4d27281c561c314a2947a8c Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 25 Aug 2024 11:16:34 +0800 Subject: [PATCH 07/13] fix: resolve conflict --- build.sh | 2 +- save_load.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 53532ea2..644a5df6 100755 --- a/build.sh +++ b/build.sh @@ -50,7 +50,7 @@ do --) shift break - ;; + ;; esac shift done diff --git a/save_load.sh b/save_load.sh index 435dc48e..09f36c26 100644 --- a/save_load.sh +++ b/save_load.sh @@ -15,4 +15,4 @@ redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset redis-cli -p 7777 raft.node dosnapshot redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset -redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 \ No newline at end of file +redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 From f409652f9f4e9569e9b9579a0753db04a894a02c Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 25 Aug 2024 11:22:24 +0800 Subject: [PATCH 08/13] fix: resolve conflict --- save_load.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/save_load.sh b/save_load.sh index 09f36c26..a45170dc 100644 --- a/save_load.sh +++ b/save_load.sh @@ -3,8 +3,13 @@ killall -9 pikiwidb mkdir leader follower1 -cd leader && ulimit -n 99999 && rm -fr * && ../bin/pikiwidb ../pikiwidb.conf --port 7777 & -cd follower1 && ulimit -n 99999 && rm -fr * && ../bin/pikiwidb ../pikiwidb.conf --port 8888 & + +PWD=`pwd` +PROJECT_HOME="${PWD}/../" +BIN="${PROJECT_HOME}/bin/pikiwidb" +CONF="${PROJECT_HOME}/etc/conf/pikiwidb.conf" +cd leader && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 7777 & +cd follower1 && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 8888 & sleep 5 redis-cli -p 7777 raft.cluster init From 9ec19206af0a2947b33011db96fc646a4021b4e9 Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 25 Aug 2024 11:27:17 +0800 Subject: [PATCH 09/13] fix: resolve conflict --- save_load.sh => etc/script/save_load.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename save_load.sh => etc/script/save_load.sh (100%) diff --git a/save_load.sh b/etc/script/save_load.sh similarity index 100% rename from save_load.sh rename to etc/script/save_load.sh From 70c63421c1f2a7cd38a83cf3670e7bd4a5f8423f Mon Sep 17 00:00:00 2001 From: panlei-coder Date: Sun, 25 Aug 2024 11:28:26 +0800 Subject: [PATCH 10/13] fix: resolve conflict --- etc/script/save_load.sh | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 etc/script/save_load.sh diff --git a/etc/script/save_load.sh b/etc/script/save_load.sh deleted file mode 100644 index a45170dc..00000000 --- a/etc/script/save_load.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -killall -9 pikiwidb -mkdir leader follower1 - - -PWD=`pwd` -PROJECT_HOME="${PWD}/../" -BIN="${PROJECT_HOME}/bin/pikiwidb" -CONF="${PROJECT_HOME}/etc/conf/pikiwidb.conf" -cd leader && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 7777 & -cd follower1 && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 8888 & -sleep 5 - -redis-cli -p 7777 raft.cluster init - -redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset -redis-cli -p 7777 raft.node dosnapshot -redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset -redis-cli -p 7777 raft.node dosnapshot -redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset - -redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 From 97b6b9a0d272dd5963d4faab0fd6108d3c65dbd1 Mon Sep 17 00:00:00 2001 From: wuyihan <18898762098@163.com> Date: Sun, 10 Nov 2024 22:06:30 +0800 Subject: [PATCH 11/13] resolve conflicts --- cmake/braft.cmake | 3 +- etc/script/save_load.sh | 23 +++ src/pikiwidb.cc | 301 ---------------------------------------- 3 files changed, 25 insertions(+), 302 deletions(-) create mode 100755 etc/script/save_load.sh delete mode 100644 src/pikiwidb.cc diff --git a/cmake/braft.cmake b/cmake/braft.cmake index 79bf4e89..8e89ad65 100644 --- a/cmake/braft.cmake +++ b/cmake/braft.cmake @@ -20,7 +20,8 @@ ExternalProject_Add( DEPENDS brpc # The pr on braft is not merged, so I am using my own warehouse to run the test for the time being GIT_REPOSITORY "https://github.com/pikiwidb/braft.git" - GIT_TAG v1.1.2.1 + GIT_TAG v1.1.2-alpha2 + URL_HASH SHA256=6afed189e97b7e6bf5864c5162fab3365b07d515fe0de4c1b0d61eff96cf772f GIT_SHALLOW true CMAKE_ARGS -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE} diff --git a/etc/script/save_load.sh b/etc/script/save_load.sh new file mode 100755 index 00000000..c3834234 --- /dev/null +++ b/etc/script/save_load.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +killall -9 kiwi +mkdir leader follower1 + + +PWD=`pwd` +PROJECT_HOME="${PWD}/../" +BIN="${PROJECT_HOME}/bin/kiwi" +CONF="${PROJECT_HOME}/etc/conf/kiwi.conf" +cd leader && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 7777 & +cd follower1 && ulimit -n 99999 && rm -fr * && ${BIN} ${CONF} --port 8888 & +sleep 5 + +redis-cli -p 7777 raft.cluster init + +redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset +redis-cli -p 7777 raft.node dosnapshot +redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset +redis-cli -p 7777 raft.node dosnapshot +redis-benchmark -p 7777 -c 5 -n 10000 -r 10000 -d 1024 -t hset + +redis-cli -p 8888 raft.cluster join 127.0.0.1:7777 diff --git a/src/pikiwidb.cc b/src/pikiwidb.cc deleted file mode 100644 index 3bf60c5e..00000000 --- a/src/pikiwidb.cc +++ /dev/null @@ -1,301 +0,0 @@ -/* - * Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ - -// -// PikiwiDB.cc - -#include "pikiwidb.h" - -#include -#include -#include -#include -#include -#include - -#include "praft/praft.h" -#include "pstd/log.h" -#include "pstd/pstd_util.h" - -#include "client.h" -#include "config.h" -#include "helper.h" -#include "pikiwidb_logo.h" -#include "slow_log.h" -#include "store.h" - -std::unique_ptr g_pikiwidb; -using namespace pikiwidb; - -static void IntSigHandle(const int sig) { - INFO("Catch Signal {}, cleanup...", sig); - g_pikiwidb->Stop(); -} - -static void SignalSetup() { - signal(SIGHUP, SIG_IGN); - signal(SIGPIPE, SIG_IGN); - signal(SIGINT, &IntSigHandle); - signal(SIGQUIT, &IntSigHandle); - signal(SIGTERM, &IntSigHandle); -} - -const uint32_t PikiwiDB::kRunidSize = 40; - -static void Usage() { - std::cerr << "Usage: ./pikiwidb-server [/path/to/redis.conf] [options]\n\ - ./pikiwidb-server -v or --version\n\ - ./pikiwidb-server -h or --help\n\ -Examples:\n\ - ./pikiwidb-server (run the server with default conf)\n\ - ./pikiwidb-server /etc/redis/6379.conf\n\ - ./pikiwidb-server --port 7777\n\ - ./pikiwidb-server --port 7777 --slaveof 127.0.0.1 8888\n\ - ./pikiwidb-server /etc/myredis.conf --loglevel verbose\n"; -} - -bool PikiwiDB::ParseArgs(int ac, char* av[]) { - for (int i = 0; i < ac; i++) { - if (cfg_file_.empty() && ::access(av[i], R_OK) == 0) { - cfg_file_ = av[i]; - continue; - } else if (strncasecmp(av[i], "-v", 2) == 0 || strncasecmp(av[i], "--version", 9) == 0) { - std::cerr << "PikiwiDB Server version: " << KPIKIWIDB_VERSION << " bits=" << (sizeof(void*) == 8 ? 64 : 32) - << std::endl; - std::cerr << "PikiwiDB Server Build Type: " << KPIKIWIDB_BUILD_TYPE << std::endl; -#if defined(KPIKIWIDB_BUILD_DATE) - std::cerr << "PikiwiDB Server Build Date: " << KPIKIWIDB_BUILD_DATE << std::endl; -#endif -#if defined(KPIKIWIDB_GIT_COMMIT_ID) - std::cerr << "PikiwiDB Server Build GIT SHA: " << KPIKIWIDB_GIT_COMMIT_ID << std::endl; -#endif - - exit(0); - } else if (strncasecmp(av[i], "-h", 2) == 0 || strncasecmp(av[i], "--help", 6) == 0) { - Usage(); - exit(0); - } else if (strncasecmp(av[i], "--port", 6) == 0) { - if (++i == ac) { - return false; - } - port_ = static_cast(std::atoi(av[i])); - } else if (strncasecmp(av[i], "--loglevel", 10) == 0) { - if (++i == ac) { - return false; - } - log_level_ = std::string(av[i]); - } else if (strncasecmp(av[i], "--slaveof", 9) == 0) { - if (i + 2 >= ac) { - return false; - } - - master_ = std::string(av[++i]); - master_port_ = static_cast(std::atoi(av[++i])); - } else { - std::cerr << "Unknow option " << av[i] << std::endl; - return false; - } - } - - return true; -} - -void PikiwiDB::OnNewConnection(pikiwidb::TcpConnection* obj) { - INFO("New connection from {}:{}", obj->GetPeerIP(), obj->GetPeerPort()); - - auto client = std::make_shared(obj); - obj->SetContext(client); - - client->OnConnect(); - - auto msg_cb = std::bind(&pikiwidb::PClient::HandlePackets, client.get(), std::placeholders::_1, std::placeholders::_2, - std::placeholders::_3); - obj->SetMessageCallback(msg_cb); - obj->SetOnDisconnect([](pikiwidb::TcpConnection* obj) { - INFO("disconnect from {}", obj->GetPeerIP()); - obj->GetContext()->SetState(pikiwidb::ClientState::kClosed); - }); - obj->SetNodelay(true); - obj->SetEventLoopSelector([this]() { return worker_threads_.ChooseNextWorkerEventLoop(); }); - obj->SetSlaveEventLoopSelector([this]() { return slave_threads_.ChooseNextWorkerEventLoop(); }); -} - -bool PikiwiDB::Init() { - char runid[kRunidSize + 1] = ""; - getRandomHexChars(runid, kRunidSize); - g_config.Set("runid", {runid, kRunidSize}, true); - - if (port_ != 0) { - g_config.Set("port", std::to_string(port_), true); - } - - if (!log_level_.empty()) { - g_config.Set("log-level", log_level_, true); - } - - NewTcpConnectionCallback cb = std::bind(&PikiwiDB::OnNewConnection, this, std::placeholders::_1); - if (!worker_threads_.Init(g_config.ip.ToString().c_str(), g_config.port.load(), cb)) { - ERROR("worker_threads Init failed. IP = {} Port = {}", g_config.ip.ToString(), g_config.port.load()); - return false; - } - - auto num = g_config.worker_threads_num.load() + g_config.slave_threads_num.load(); - auto kMaxWorkerNum = IOThreadPool::GetMaxWorkerNum(); - if (num > kMaxWorkerNum) { - ERROR("number of threads can't exceeds {}, now is {}", kMaxWorkerNum, num); - return false; - } - worker_threads_.SetWorkerNum(static_cast(g_config.worker_threads_num.load())); - slave_threads_.SetWorkerNum(static_cast(g_config.slave_threads_num.load())); - - // now we only use fast cmd thread pool - auto status = cmd_threads_.Init(g_config.fast_cmd_threads_num.load(), 0, "pikiwidb-cmd"); - if (!status.ok()) { - ERROR("init cmd thread pool failed: {}", status.ToString()); - return false; - } - - PSTORE.Init(g_config.databases.load(std::memory_order_relaxed)); - - PSlowLog::Instance().SetThreshold(g_config.slow_log_time.load()); - PSlowLog::Instance().SetLogLimit(static_cast(g_config.slow_log_max_len.load())); - - // init base loop - auto loop = worker_threads_.BaseLoop(); - loop->ScheduleRepeatedly(1000, &PReplication::Cron, &PREPL); - - // master ip - if (!g_config.ip.empty()) { - PREPL.SetMasterAddr(g_config.master_ip.ToString().c_str(), g_config.master_port.load()); - } - - return true; -} - -void PikiwiDB::Run() { - worker_threads_.SetName("pikiwi-main"); - slave_threads_.SetName("pikiwi-slave"); - - cmd_threads_.Start(); - - std::thread t([this]() { - auto slave_loop = slave_threads_.BaseLoop(); - slave_loop->Init(); - slave_threads_.Run(0, nullptr); - }); - - worker_threads_.Run(0, nullptr); - - if (t.joinable()) { - t.join(); // wait for slave thread exit - } - INFO("server exit running"); -} - -void PikiwiDB::Stop() { - pikiwidb::PRAFT.ShutDown(); - pikiwidb::PRAFT.Join(); - pikiwidb::PRAFT.Clear(); - slave_threads_.Exit(); - worker_threads_.Exit(); - cmd_threads_.Stop(); -} - -// pikiwidb::CmdTableManager& PikiwiDB::GetCmdTableManager() { return cmd_table_manager_; } - -static void InitLogs() { - logger::Init("logs/pikiwidb_server.log"); - -#if BUILD_DEBUG - spdlog::set_level(spdlog::level::debug); -#else - spdlog::set_level(spdlog::level::info); -#endif -} - -static int InitLimit() { - rlimit limit; - rlim_t maxfiles = g_config.max_clients; - if (getrlimit(RLIMIT_NOFILE, &limit) == -1) { - WARN("getrlimit error: {}", strerror(errno)); - } else if (limit.rlim_cur < maxfiles) { - rlim_t old_limit = limit.rlim_cur; - limit.rlim_cur = maxfiles; - limit.rlim_max = maxfiles; - if (setrlimit(RLIMIT_NOFILE, &limit) != -1) { - WARN("your 'limit -n ' of {} is not enough for PikiwiDB to start. PikiwiDB have successfully reconfig it to {}", - old_limit, limit.rlim_cur); - } else { - ERROR( - "your 'limit -n ' of {} is not enough for PikiwiDB to start." - " PikiwiDB can not reconfig it({}), do it by yourself", - old_limit, strerror(errno)); - return -1; - } - } - return 0; -} - -static void daemonize() { - if (fork()) { - exit(0); /* parent exits */ - } - setsid(); /* create a new session */ -} - -static void closeStd() { - int fd; - fd = open("/dev/null", O_RDWR, 0); - if (fd != -1) { - dup2(fd, STDIN_FILENO); - dup2(fd, STDOUT_FILENO); - dup2(fd, STDERR_FILENO); - close(fd); - } -} - -int main(int ac, char* av[]) { - g_pikiwidb = std::make_unique(); - if (!g_pikiwidb->ParseArgs(ac - 1, av + 1)) { - Usage(); - return -1; - } - - if (!g_pikiwidb->GetConfigName().empty()) { - if (!g_config.LoadFromFile(g_pikiwidb->GetConfigName())) { - std::cerr << "Load config file [" << g_pikiwidb->GetConfigName() << "] failed!\n"; - return -1; - } - } - - // output logo to console - char logo[512] = ""; - snprintf(logo, sizeof logo - 1, pikiwidbLogo, KPIKIWIDB_VERSION, static_cast(sizeof(void*)) * 8, - static_cast(g_config.port)); - std::cout << logo; - - if (g_config.daemonize.load()) { - daemonize(); - } - - // InitLimit(); // Code problem, program startup direct segment error - pstd::InitRandom(); - SignalSetup(); - InitLogs(); - - if (g_config.daemonize.load()) { - closeStd(); - } - - if (g_pikiwidb->Init()) { - g_pikiwidb->Run(); - } - - // when process exit, flush log - spdlog::get(logger::Logger::Instance().Name())->flush(); - return 0; -} From f8063ad133ef6d7495b3cbb19cbffd776c2477fe Mon Sep 17 00:00:00 2001 From: liuyuecai Date: Sat, 16 Nov 2024 18:29:40 +0800 Subject: [PATCH 12/13] reformat code --- src/proto_parser.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) mode change 100755 => 100644 src/proto_parser.cc diff --git a/src/proto_parser.cc b/src/proto_parser.cc old mode 100755 new mode 100644 index c0e78446..b13d6e3c --- a/src/proto_parser.cc +++ b/src/proto_parser.cc @@ -26,7 +26,6 @@ void PProtoParser::Reset() { numOfParam_ = 0; params_.clear(); - } PParseResult PProtoParser::ParseRequest(const char*& ptr, const char* end) { @@ -96,7 +95,7 @@ PParseResult PProtoParser::parseStrval(const char*& ptr, const char* end, PStrin assert(paramLen_ >= 0); if (static_cast(end - ptr) < paramLen_ + 2) { - paramLen_-=(end-ptr); + paramLen_ -= (end - ptr); result.append(ptr, end - ptr); return PParseResult::kWait; } From 1c9e6bbea4eb74afd7d72fb1792890f7c9796d60 Mon Sep 17 00:00:00 2001 From: liuyuecai Date: Sat, 16 Nov 2024 19:27:52 +0800 Subject: [PATCH 13/13] change braft version --- cmake/braft.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/braft.cmake b/cmake/braft.cmake index c9bb2268..01c596ac 100644 --- a/cmake/braft.cmake +++ b/cmake/braft.cmake @@ -20,8 +20,8 @@ ExternalProject_Add( DEPENDS brpc # The pr on braft is not merged, so I am using my own warehouse to run the test for the time being GIT_REPOSITORY "https://github.com/pikiwidb/braft.git" - GIT_TAG v1.1.2-alpha2 - URL_HASH SHA256=6afed189e97b7e6bf5864c5162fab3365b07d515fe0de4c1b0d61eff96cf772f + GIT_TAG v1.1.2.1 + URL_HASH SHA256=d41883529b55266367d385d40f638967a0391d8a GIT_SHALLOW true CMAKE_ARGS -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE}