Revert "#1870 change storage usage calculation]"

This reverts commit 2a37352.
skalenetwork · Oct 29, 2024 · f19aa68 · f19aa68
1 parent 2a37352
commit f19aa68
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 53 deletions.
diff --git a/libbatched-io/BatchedRotatingHistoricDbIO.cpp b/libbatched-io/BatchedRotatingHistoricDbIO.cpp
@@ -33,6 +33,7 @@ void BatchedRotatingHistoricDbIO::rotate( uint64_t blockNumber ) {
     std::lock_guard< std::mutex > lock( mutex );
     assert( blockNumber > blockNumbers.back() );
 
+    auto storageUsed = currentPiece()->lookup( dev::db::Slice( "storageUsed" ) );
     currentPiece()->kill( dev::db::Slice( "storageUsed" ) );
 
     // move current to used

diff --git a/libhistoric/HistoricState.cpp b/libhistoric/HistoricState.cpp
@@ -222,9 +222,27 @@ void HistoricState::clearCacheIfTooLarge() const {
     }
 }
 
+uint64_t HistoricState::calculateNewDataSize( const AccountMap& _cache ) const {
+    uint64_t res = 0;
+    for ( const auto& item : _cache ) {
+        if ( item.second.isDirty() )
+            if ( item.second.isAlive() ) {
+                res += 32 * 2 * item.second.storageOverlay().size();  // for storage
+                if ( item.second.hasNewCode() )
+                    res += 32 + item.second.code().size();           // for new code
+                res += 32 * ( item.second.version() != 0 ? 5 : 4 );  // for account details
+                res += 20;                                           // for address
+            }
+    }
+
+    return res;
+}
+
 void HistoricState::commitExternalChanges( AccountMap const& _accountMap, uint64_t _blockNumber ) {
     auto historicStateStart = dev::db::LevelDB::getCurrentTimeMs();
+    auto newDataSize = calculateNewDataSize( _accountMap );
     commitExternalChangesIntoTrieDB( _accountMap, m_state );
+    updateStorageUsage( newDataSize );
     m_state.db()->commit( std::to_string( _blockNumber ), true );
     m_changeLog.clear();
     m_cache.clear();
@@ -839,7 +857,6 @@ void HistoricState::rotateDbsIfNeeded( uint64_t _blockNumber ) {
         return;
     if ( m_db.storageUsed() > m_maxHistoricStateDbSize ) {
         m_rotatingTreeDb->rotate( _blockNumber );
-        m_db.updateStorageUsage( 0 );
         m_storageUsage = 0;
     }
 }
diff --git a/libhistoric/HistoricState.h b/libhistoric/HistoricState.h
@@ -355,6 +355,13 @@ class HistoricState {
     AddressHash commitExternalChangesIntoTrieDB(
         AccountMap const& _cache, SecureTrieDB< Address, skale::OverlayDB >& _state );
 
+    uint64_t calculateNewDataSize( AccountMap const& _cache ) const;
+
+    void updateStorageUsage( uint64_t bytes ) {
+        m_storageUsage += bytes;
+        m_db.updateStorageUsage( m_storageUsage );
+    }
+
     dev::s256 storageUsedTotal() const { return m_db.storageUsed(); }
 
     uint64_t m_totalTimeSpentInStateCommitsPerBlock = 0;

diff --git a/libskale/OverlayDB.cpp b/libskale/OverlayDB.cpp
@@ -261,8 +261,6 @@ void OverlayDB::commit( const std::string& _debugCommitId, bool _isHistoricState
             m_cache.clear();
             m_auxiliaryCache.clear();
             m_storageCache.clear();
-            m_historicMain.clear();
-            m_historicAux.clear();
             m_db_face->revert();
         }
     } else {

diff --git a/test/unittests/libethereum/StateUnitTests.cpp b/test/unittests/libethereum/StateUnitTests.cpp
@@ -19,7 +19,6 @@
 /// @file
 /// State unit tests.
 
-#include <libdevcore/FileSystem.h>
 #include <libdevcore/TransientDirectory.h>
 #include <libethcore/BasicAuthority.h>
 #include <libethereum/Block.h>
@@ -212,15 +211,6 @@ class DbRotationFixture : public TestOutputHelperFixture {
         auto di = directory_iterator(m_tempDirState.path() + "/historic_state/00000000/state");
         return std::distance(begin(di), end(di));
     }
-    uint64_t storageUsage() {
-        uint64_t total = 0;
-        state.mutableHistoricState().db().db()->forEach( [&total]( const dev::db::Slice& _key, const dev::db::Slice& _value ) {
-            std::cout << dev::toHex( _key ) << ' ' << dev::toHex( _value ) << '\n';
-            total += _key.size() + _value.size();
-            return true;
-        } );
-        return total;
-    }
 };
 
 BOOST_FIXTURE_TEST_SUITE( DbRotationSuite, DbRotationFixture )
@@ -402,46 +392,6 @@ BOOST_AUTO_TEST_CASE( updateStorage ) {
     BOOST_CHECK_EQUAL(countDbPieces(), 3);
 }
 
-BOOST_AUTO_TEST_CASE( storageUsed ) {
-    h256 location = h256(123456);
-
-    State sw = state.createStateModifyCopy();
-
-    for (size_t i = 0; i < 100; ++i) {
-        sw.incNonce(address1);
-        sw.commit( dev::eth::CommitBehaviour::RemoveEmptyAccounts, 1001 + 4 * i );
-        sw.mutableHistoricState().saveRootForBlockNumber( 1001 + 4 * i );
-
-        sw.setStorage(address1, location, 3 * i + 1);
-        sw.commit( dev::eth::CommitBehaviour::RemoveEmptyAccounts, 1002 + 4 * i );
-        sw.mutableHistoricState().saveRootForBlockNumber( 1002 + 4 * i );
-
-        sw.setStorage(address1, location, 3 * i + 2);
-        sw.commit( dev::eth::CommitBehaviour::RemoveEmptyAccounts, 1003 + 4 * i );
-        sw.mutableHistoricState().saveRootForBlockNumber( 1003 + 4 * i );
-
-        bytes code( 50 );
-        srand( std::time( 0 ) + i );
-        for ( auto& b : code ) {
-            b = rand();
-        }
-        dev::Address address = dev::Address::random();
-        sw.setCode( address, code, 3 * i + 3 );
-        sw.commit( dev::eth::CommitBehaviour::RemoveEmptyAccounts, 1004 + 4 * i );
-        sw.mutableHistoricState().saveRootForBlockNumber( 1004 + 4 * i );
-
-        state.mutableHistoricState().setRoot(sw.mutableHistoricState().globalRoot(), sw.mutableHistoricState().globalRootBlockNumber());
-    }
-
-    auto storageUsageReal = storageUsage();
-    auto storageUsageFromDB = state.mutableHistoricState().db().storageUsed();
-
-    std::cout << dev::getDirSize( m_tempDirState.path() / boost::filesystem::path( "historic_state" ) ) << '\n';
-
-    BOOST_CHECK_GE( storageUsageFromDB, storageUsageReal );
-    BOOST_CHECK_LE( double(storageUsageFromDB), 1.1 * storageUsageReal );
-}
-
 BOOST_AUTO_TEST_SUITE_END()
 
 BOOST_AUTO_TEST_SUITE_END()