Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bug/is 968 functional tests #2030

Merged
merged 5 commits into from
Oct 23, 2024
Merged

Bug/is 968 functional tests #2030

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e7bc6ea
IS 968 fix functional tests for historic build
olehnikolaiev Oct 21, 2024
e4d710a
IS 968 fix functional tests for historic build
olehnikolaiev Oct 21, 2024
a5492a9
IS 968 fix functional tests for historic build
olehnikolaiev Oct 22, 2024
cefd6af
IS 968 fix snapshot dowloading for small snapshots
olehnikolaiev Oct 23, 2024
ab180dc
IS 968 rename timeout
olehnikolaiev Oct 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions libskale/SnapshotManager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,11 +160,11 @@ void SnapshotManager::restoreSnapshot( unsigned _blockNumber ) {

UnsafeRegion::lock ur_lock;

std::vector< std::string > volumes;
if ( chainParams.nodeInfo.archiveMode && _blockNumber == 0 )
volumes = coreVolumes;
else
std::vector< std::string > volumes = coreVolumes;
#ifdef HISTORIC_STATE
if ( _blockNumber > 0 )
volumes = allVolumes;
#endif

int dummy_counter = 0;
for ( const string& vol : volumes ) {
Expand All @@ -180,6 +180,19 @@ void SnapshotManager::restoreSnapshot( unsigned _blockNumber ) {
batched_io::test_crash_before_commit( "SnapshotManager::doSnapshot" );

} // for

if ( _blockNumber == 0 ) {
#ifdef HISTORIC_STATE
for ( const string& vol : allVolumes ) {
// continue if already present
if ( fs::exists( dataDir / vol ) && 0 == btrfs.present( ( dataDir / vol ).c_str() ) )
continue;

// create if not created yet ( only makes sense for historic nodes and 0 block number )
btrfs.subvolume.create( ( dataDir / vol ).c_str() );
} // for
#endif
}
}

// exceptions:
Expand Down Expand Up @@ -747,11 +760,11 @@ void SnapshotManager::computeSnapshotHash( unsigned _blockNumber, bool is_checki

int dummy_counter = 0;

std::vector< std::string > volumes;
if ( chainParams.nodeInfo.archiveMode && _blockNumber == 0 )
volumes = coreVolumes;
else
std::vector< std::string > volumes = coreVolumes;
#ifdef HISTORIC_STATE
if ( _blockNumber > 0 )
volumes = allVolumes;
#endif

for ( const auto& volume : volumes ) {
int res = btrfs.subvolume.property_set(
Expand Down
4 changes: 1 addition & 3 deletions libweb3jsonrpc/Skale.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -582,8 +582,7 @@ std::string Skale::oracle_checkResult( std::string& receipt ) {
namespace snapshot {

bool download( const std::string& strURLWeb3, unsigned& block_number, const fs::path& saveTo,
fn_progress_t onProgress, bool isBinaryDownload, std::string* pStrErrorDescription,
bool forArchiveNode ) {
fn_progress_t onProgress, bool isBinaryDownload, std::string* pStrErrorDescription ) {
if ( pStrErrorDescription )
pStrErrorDescription->clear();
std::ofstream f;
Expand Down Expand Up @@ -636,7 +635,6 @@ bool download( const std::string& strURLWeb3, unsigned& block_number, const fs::
joIn["method"] = "skale_getSnapshot";
nlohmann::json joParams = nlohmann::json::object();
joParams["blockNumber"] = block_number;
joParams["forArchiveNode"] = forArchiveNode;
joIn["params"] = joParams;
skutils::rest::data_t d = cli.call( joIn );
if ( !d.err_s_.empty() ) {
Expand Down
2 changes: 1 addition & 1 deletion libweb3jsonrpc/Skale.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ typedef std::function< bool( size_t idxChunck, size_t cntChunks ) > fn_progress_

extern bool download( const std::string& strURLWeb3, unsigned& block_number, const fs::path& saveTo,
fn_progress_t onProgress, bool isBinaryDownload = true,
std::string* pStrErrorDescription = nullptr, bool forArchiveNode = false );
std::string* pStrErrorDescription = nullptr );

}; // namespace snapshot

Expand Down
25 changes: 10 additions & 15 deletions skaled/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,8 +251,7 @@ void downloadSnapshot( unsigned block_number, std::shared_ptr< SnapshotManager >
const std::string& strURLWeb3, const ChainParams& chainParams ) {
fs::path saveTo;
try {
clog( VerbosityInfo, "downloadSnapshot" )
<< cc::normal( "Will download snapshot from " ) << cc::u( strURLWeb3 ) << std::endl;
clog( VerbosityInfo, "downloadSnapshot" ) << "Will download snapshot from " << strURLWeb3;

try {
bool isBinaryDownload = true;
Expand All @@ -262,11 +261,10 @@ void downloadSnapshot( unsigned block_number, std::shared_ptr< SnapshotManager >
strURLWeb3, block_number, saveTo,
[&]( size_t idxChunck, size_t cntChunks ) -> bool {
clog( VerbosityInfo, "downloadSnapshot" )
<< cc::normal( "... download progress ... " ) << cc::size10( idxChunck )
<< cc::normal( " of " ) << cc::size10( cntChunks ) << "\r";
<< "... download progress ... " << idxChunck << " of " << cntChunks << "\r";
return true; // continue download
},
isBinaryDownload, &strErrorDescription, chainParams.nodeInfo.archiveMode );
isBinaryDownload, &strErrorDescription );
std::cout << " \r"; // clear
// progress
// line
Expand All @@ -281,14 +279,12 @@ void downloadSnapshot( unsigned block_number, std::shared_ptr< SnapshotManager >
std::throw_with_nested( std::runtime_error( "Exception while downloading snapshot" ) );
}
clog( VerbosityInfo, "downloadSnapshot" )
<< cc::success( "Snapshot download success for block " )
<< cc::u( to_string( block_number ) ) << std::endl;
<< "Snapshot download success for block " << to_string( block_number );
try {
snapshotManager->importDiff( block_number );
} catch ( ... ) {
std::throw_with_nested( std::runtime_error(
cc::fatal( "FATAL:" ) + " " +
cc::error( "Exception while importing downloaded snapshot: " ) ) );
std::throw_with_nested(
std::runtime_error( "FATAL: Exception while importing downloaded snapshot: " ) );
}

/// HACK refactor this piece of code! ///
Expand All @@ -304,7 +300,7 @@ void downloadSnapshot( unsigned block_number, std::shared_ptr< SnapshotManager >
}
if ( db_path.empty() ) {
clog( VerbosityError, "downloadSnapshot" )
<< cc::fatal( "Snapshot downloaded without " + prefix + " db" ) << std::endl;
<< "Snapshot downloaded without " + prefix + " db";
return;
}

Expand All @@ -315,8 +311,7 @@ void downloadSnapshot( unsigned block_number, std::shared_ptr< SnapshotManager >

} catch ( ... ) {
std::throw_with_nested(
std::runtime_error( cc::fatal( "FATAL:" ) + " " +
cc::error( "Exception while processing downloaded snapshot: " ) ) );
std::runtime_error( "FATAL: Exception while processing downloaded snapshot: " ) );
}
if ( !saveTo.empty() )
fs::remove( saveTo );
Expand Down Expand Up @@ -1657,9 +1652,9 @@ int main( int argc, char** argv ) try {
// sleep before send skale_getSnapshot again - will receive error
clog( VerbosityInfo, "main" )
<< std::string( "Will sleep for " )
<< chainParams.sChain.snapshotDownloadInactiveTimeout
<< chainParams.sChain.snapshotDownloadInactiveTimeout + 10
<< std::string( " seconds before downloading 0 snapshot" );
sleep( chainParams.sChain.snapshotDownloadInactiveTimeout );
sleep( chainParams.sChain.snapshotDownloadInactiveTimeout + 10 );

downloadAndProccessSnapshot(
snapshotManager, chainParams, urlToDownloadSnapshotFrom, false );
Expand Down
Loading