Merge pull request #1684 from skalenetwork/develop

New beta
skalenetwork · Oct 2, 2023 · 6c4e168 · 6c4e168
2 parents 98e5ea5 + 5b3ccae
commit 6c4e168
Show file tree

Hide file tree

Showing 14 changed files with 107 additions and 18 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -8,7 +8,15 @@ on:
 defaults:
   run:
     shell: bash
-jobs: 
+jobs:
+  cancel-runs:
+    name: Cancel Previous Runs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.9.1
+        with:
+          access_token: ${{ github.token }}
   build:
     runs-on: self-hosted
     env:

diff --git a/README.md b/README.md
@@ -58,7 +58,7 @@ If you have already cloned the repo and forgot to pass `--recurse-submodules`, e
 
 ```
 sudo apt update
-sudo apt install autoconf build-essential cmake libprocps-dev libtool texinfo wget yasm flex bison btrfs-progs python python3-pip gawk git vim doxygen 
+sudo apt install autoconf build-essential cmake libprocps-dev libtool texinfo wget yasm flex bison btrfs-progs python3 python3-pip gawk git vim doxygen 
 sudo apt install make build-essential cmake pkg-config libgnutls28-dev libssl-dev unzip zlib1g-dev libgcrypt20-dev docker.io gcc-9 g++-9 gperf clang-format-11 gnutls-dev
 sudo apt install nettle-dev libhiredis-dev redis-server google-perftools libgoogle-perftools-dev lcov
 ```

diff --git a/libdevcore/LevelDB.cpp b/libdevcore/LevelDB.cpp
@@ -70,6 +70,7 @@ class LevelDBWriteBatch : public WriteBatchFace {
 
 private:
     leveldb::WriteBatch m_writeBatch;
+    std::atomic< uint64_t > keysToBeDeletedCount;
 };
 
 void LevelDBWriteBatch::insert( Slice _key, Slice _value ) {
@@ -78,6 +79,7 @@ void LevelDBWriteBatch::insert( Slice _key, Slice _value ) {
 }
 
 void LevelDBWriteBatch::kill( Slice _key ) {
+    LevelDB::g_keysToBeDeletedStats++;
     m_writeBatch.Delete( toLDBSlice( _key ) );
 }
 
@@ -168,6 +170,9 @@ void LevelDB::insert( Slice _key, Slice _value ) {
 void LevelDB::kill( Slice _key ) {
     leveldb::Slice const key( _key.data(), _key.size() );
     auto const status = m_db->Delete( m_writeOptions, key );
+    // At this point the key is not actually deleted. It will be deleted when the batch
+    // is committed
+    g_keysToBeDeletedStats++;
     checkStatus( status );
 }
 
@@ -185,6 +190,10 @@ void LevelDB::commit( std::unique_ptr< WriteBatchFace > _batch ) {
             DatabaseError() << errinfo_comment( "Invalid batch type passed to LevelDB::commit" ) );
     }
     auto const status = m_db->Write( m_writeOptions, &batchPtr->writeBatch() );
+    // Commit happened. This means the keys actually got deleted in LevelDB. Increment key deletes
+    // stats and set g_keysToBeDeletedStats to zero
+    g_keyDeletesStats += g_keysToBeDeletedStats;
+    g_keysToBeDeletedStats = 0;
     checkStatus( status );
 }
 
@@ -275,5 +284,12 @@ void LevelDB::doCompaction() const {
     m_db->CompactRange( nullptr, nullptr );
 }
 
+std::atomic< uint64_t > LevelDB::g_keysToBeDeletedStats = 0;
+std::atomic< uint64_t > LevelDB::g_keyDeletesStats = 0;
+
+uint64_t LevelDB::getKeyDeletesStats() {
+    return g_keyDeletesStats;
+}
+
 }  // namespace db
 }  // namespace dev
diff --git a/libdevcore/LevelDB.h b/libdevcore/LevelDB.h
@@ -61,6 +61,13 @@ class LevelDB : public DatabaseFace {
 
     void doCompaction() const;
 
+    // Return the total count of key deletes  since the start
+    static uint64_t getKeyDeletesStats();
+    // count of the keys that were deleted since the start of skaled
+    static std::atomic< uint64_t > g_keyDeletesStats;
+    // count of the keys that are scheduled to be deleted but are not yet deleted
+    static std::atomic< uint64_t > g_keysToBeDeletedStats;
+
 private:
     std::unique_ptr< leveldb::DB > m_db;
     leveldb::ReadOptions const m_readOptions;

diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp
@@ -43,6 +43,7 @@
 #include <libdevcore/microprofile.h>
 
 #include <libdevcore/FileSystem.h>
+#include <libdevcore/LevelDB.h>
 #include <libdevcore/system_usage.h>
 
 #ifdef HISTORIC_STATE
@@ -152,18 +153,18 @@ Client::Client( ChainParams const& _params, int _networkID,
 
     init( _forceAction, _networkID );
 
+    // Set timestamps for patches
     TotalStorageUsedPatch::g_client = this;
-    ContractStorageLimitPatch::contractStoragePatchTimestamp =
-        chainParams().sChain.contractStoragePatchTimestamp;
-    ContractStorageZeroValuePatch::contractStorageZeroValuePatchTimestamp =
-        chainParams().sChain.contractStorageZeroValuePatchTimestamp;
-    VerifyDaSigsPatch::verifyDaSigsPatchTimestamp = chainParams().sChain.verifyDaSigsPatchTimestamp;
-    RevertableFSPatch::revertableFSPatchTimestamp = chainParams().sChain.revertableFSPatchTimestamp;
-    StorageDestructionPatch::storageDestructionPatchTimestamp =
-        chainParams().sChain.storageDestructionPatchTimestamp;
-    POWCheckPatch::powCheckPatchTimestamp = chainParams().sChain.powCheckPatchTimestamp;
+    ContractStorageLimitPatch::setTimestamp( chainParams().sChain.contractStoragePatchTimestamp );
+    ContractStorageZeroValuePatch::setTimestamp(
+        chainParams().sChain.contractStorageZeroValuePatchTimestamp );
+    VerifyDaSigsPatch::setTimestamp( chainParams().sChain.verifyDaSigsPatchTimestamp );
+    RevertableFSPatch::setTimestamp( chainParams().sChain.revertableFSPatchTimestamp );
+    StorageDestructionPatch::setTimestamp( chainParams().sChain.storageDestructionPatchTimestamp );
+    POWCheckPatch::setTimestamp( chainParams().sChain.powCheckPatchTimestamp );
 }
 
+
 Client::~Client() {
     stopWorking();
 }
@@ -924,7 +925,8 @@ void Client::sealUnconditionally( bool submitToBlockChain ) {
                  << ":BDS:" << BlockDetails::howMany() << ":TSS:" << TransactionSkeleton::howMany()
                  << ":UTX:" << TransactionQueue::UnverifiedTransaction::howMany()
                  << ":VTX:" << TransactionQueue::VerifiedTransaction::howMany()
-                 << ":CMM:" << bc().getTotalCacheMemory();
+                 << ":CMM:" << bc().getTotalCacheMemory()
+                 << ":KDS:" << db::LevelDB::getKeyDeletesStats();
     if ( number() % 1000 == 0 ) {
         ssBlockStats << ":RAM:" << getRAMUsage();
         ssBlockStats << ":CPU:" << getCPUUsage();

diff --git a/libethereum/SchainPatch.h b/libethereum/SchainPatch.h
@@ -1,6 +1,17 @@
 #ifndef SCHAINPATCH_H
 #define SCHAINPATCH_H
 
-class SchainPatch {};
+#include <libdevcore/Log.h>
+
+class SchainPatch {
+public:
+    static void printInfo( const std::string& _patchName, time_t _timeStamp ) {
+        if ( _timeStamp == 0 ) {
+            cnote << "Patch " << _patchName << " is disabled";
+        } else {
+            cnote << "Patch " << _patchName << " is set at timestamp " << _timeStamp;
+        }
+    }
+};
 
 #endif  // SCHAINPATCH_H
diff --git a/libskale/ContractStorageLimitPatch.h b/libskale/ContractStorageLimitPatch.h
@@ -21,6 +21,11 @@ class ContractStorageLimitPatch : public SchainPatch {
 public:
     static bool isEnabled();
 
+    static void setTimestamp( time_t _timeStamp ) {
+        printInfo( __FILE__, _timeStamp );
+        contractStoragePatchTimestamp = _timeStamp;
+    }
+
 private:
     friend class dev::eth::Client;
     static time_t contractStoragePatchTimestamp;

diff --git a/libskale/ContractStorageZeroValuePatch.h b/libskale/ContractStorageZeroValuePatch.h
@@ -21,6 +21,12 @@ class ContractStorageZeroValuePatch : public SchainPatch {
 public:
     static bool isEnabled();
 
+    static void setTimestamp( time_t _timeStamp ) {
+        printInfo( __FILE__, _timeStamp );
+        contractStorageZeroValuePatchTimestamp = _timeStamp;
+    }
+
+
 private:
     friend class dev::eth::Client;
     static time_t contractStorageZeroValuePatchTimestamp;

diff --git a/libskale/POWCheckPatch.h b/libskale/POWCheckPatch.h
@@ -17,6 +17,11 @@ class POWCheckPatch : public SchainPatch {
 public:
     static bool isEnabled();
 
+    static void setTimestamp( time_t _timeStamp ) {
+        printInfo( __FILE__, _timeStamp );
+        powCheckPatchTimestamp = _timeStamp;
+    }
+
 private:
     friend class dev::eth::Client;
     static time_t powCheckPatchTimestamp;

diff --git a/libskale/RevertableFSPatch.h b/libskale/RevertableFSPatch.h
@@ -14,6 +14,11 @@ class RevertableFSPatch : public SchainPatch {
 public:
     static bool isEnabled();
 
+    static void setTimestamp( time_t _timeStamp ) {
+        printInfo( __FILE__, _timeStamp );
+        revertableFSPatchTimestamp = _timeStamp;
+    }
+
 private:
     friend class dev::eth::Client;
     static time_t revertableFSPatchTimestamp;

diff --git a/libskale/StorageDestructionPatch.h b/libskale/StorageDestructionPatch.h
@@ -14,6 +14,12 @@ class StorageDestructionPatch : public SchainPatch {
 public:
     static bool isEnabled();
 
+    static void setTimestamp( time_t _timeStamp ) {
+        printInfo( __FILE__, _timeStamp );
+        storageDestructionPatchTimestamp = _timeStamp;
+    }
+
+
 private:
     friend class dev::eth::Client;
     static time_t storageDestructionPatchTimestamp;

diff --git a/libskale/VerifyDaSigsPatch.h b/libskale/VerifyDaSigsPatch.h
@@ -26,6 +26,11 @@ class VerifyDaSigsPatch : public SchainPatch {
     static time_t verifyDaSigsPatchTimestamp;
     static time_t lastBlockTimestamp;
 
+    static void setTimestamp( time_t _timeStamp ) {
+        printInfo( __FILE__, _timeStamp );
+        verifyDaSigsPatchTimestamp = _timeStamp;
+    }
+
 public:
     static time_t getVerifyDaSigsPatchTimestamp();
 };

diff --git a/libweb3jsonrpc/Skale.cpp b/libweb3jsonrpc/Skale.cpp
@@ -64,13 +64,22 @@ namespace rpc {
 
 std::string exceptionToErrorMessage();
 
-Skale::Skale( Client& _client, std::shared_ptr< SharedSpace > _sharedSpace )
-    : m_client( _client ), m_shared_space( _sharedSpace ) {}
-
 volatile bool Skale::g_bShutdownViaWeb3Enabled = false;
 volatile bool Skale::g_bNodeInstanceShouldShutdown = false;
 Skale::list_fn_on_shutdown_t Skale::g_list_fn_on_shutdown;
 
+Skale::Skale( Client& _client, std::shared_ptr< SharedSpace > _sharedSpace )
+    : m_client( _client ), m_shared_space( _sharedSpace ) {}
+
+Skale::~Skale() {
+    threadExitRequested = true;
+    if ( snapshotDownloadFragmentMonitorThread != nullptr &&
+         snapshotDownloadFragmentMonitorThread->joinable() ) {
+        clog( VerbosityInfo, "Skale" ) << "Joining downloadSnapshotFragmentMonitorThread";
+        snapshotDownloadFragmentMonitorThread->join();
+    }
+}
+
 bool Skale::isWeb3ShutdownEnabled() {
     return g_bShutdownViaWeb3Enabled;
 }
@@ -199,11 +208,13 @@ nlohmann::json Skale::impl_skale_getSnapshot( const nlohmann::json& joRequest, C
                             m_client.chainParams().sChain.snapshotDownloadInactiveTimeout ) &&
                     time( NULL ) - currentSnapshotTime <
                         m_client.chainParams().sChain.snapshotDownloadTimeout ) {
-                sleep( 30 );
+                if ( threadExitRequested )
+                    break;
+                sleep( 10 );
             }
 
             clog( VerbosityInfo, "skale_downloadSnapshotFragmentMonitorThread" )
-                << "Unlocking shared space as timeout was reached.\n";
+                << "Unlocking shared space.\n";
 
             std::lock_guard< std::mutex > lock( m_snapshot_mutex );
             if ( currentSnapshotBlockNumber >= 0 ) {

diff --git a/libweb3jsonrpc/Skale.h b/libweb3jsonrpc/Skale.h
@@ -57,6 +57,7 @@ class Skale : public dev::rpc::SkaleFace {
 public:
     explicit Skale(
         dev::eth::Client& _client, std::shared_ptr< SharedSpace > _sharedSpace = nullptr );
+    virtual ~Skale();
 
     virtual RPCModules implementedModules() const override {
         return RPCModules{ RPCModule{ "skale", "0.1" } };
@@ -105,6 +106,7 @@ class Skale : public dev::rpc::SkaleFace {
     std::atomic< time_t > currentSnapshotTime = 0;
     std::atomic< time_t > lastSnapshotDownloadFragmentTime = 0;
     std::unique_ptr< std::thread > snapshotDownloadFragmentMonitorThread;
+    std::atomic_bool threadExitRequested = false;
     mutable std::mutex m_snapshot_mutex;
 };