diff --git a/NineChronicles.DataProvider.Executable/Commands/ItemEnhancementMigration.cs b/NineChronicles.DataProvider.Executable/Commands/ItemEnhancementMigration.cs new file mode 100644 index 00000000..b5023675 --- /dev/null +++ b/NineChronicles.DataProvider.Executable/Commands/ItemEnhancementMigration.cs @@ -0,0 +1,367 @@ +namespace NineChronicles.DataProvider.Executable.Commands +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using Bencodex.Types; + using Cocona; + using Lib9c.Model.Order; + using Libplanet.Action; + using Libplanet.Action.Loader; + using Libplanet.Blockchain; + using Libplanet.Blockchain.Policies; + using Libplanet.Crypto; + using Libplanet.RocksDBStore; + using Libplanet.Store; + using Libplanet.Types.Assets; + using Libplanet.Types.Blocks; + using Libplanet.Types.Tx; + using Microsoft.EntityFrameworkCore; + using Microsoft.EntityFrameworkCore.Internal; + using Microsoft.Extensions.DependencyInjection; + using MySqlConnector; + using Nekoyume; + using Nekoyume.Action; + using Nekoyume.Action.Loader; + using Nekoyume.Blockchain.Policy; + using Nekoyume.Extensions; + using Nekoyume.Helper; + using Nekoyume.Model.Item; + using Nekoyume.Model.State; + using Nekoyume.TableData; + using NineChronicles.DataProvider.DataRendering; + using NineChronicles.DataProvider.Store; + using NineChronicles.DataProvider.Store.Models; + using Serilog; + using Serilog.Events; + using static Lib9c.SerializeKeys; + + public class ItemEnhancementMigration + { + private string _connectionString; + private IStore _baseStore; + private BlockChain _baseChain; + private List _agentCheck; + private List _avatarCheck; + private MySqlStore _mySqlStore; + private BlockHash _blockHash; + private long _blockIndex; + private DateTimeOffset _blockTimeOffset; + private List _blockList; + private List _txList; + private List _agentList; + private List _avatarList; + private List _hackAndSlashList; + private List _hasWithRandomBuffList; + private List _claimStakeRewardList; + private List _runesAcquiredList; + private List _eventDungeonBattleList; + private List _eventConsumableItemCraftsList; + private List _hackAndSlashSweepList; + private List _combinationConsumableList; + private List _combinationEquipmentList; + private List _equipmentList; + private List _itemEnhancementList; + private List _buyShopEquipmentsList; + private List _buyShopCostumesList; + private List _buyShopMaterialsList; + private List _buyShopConsumablesList; + private List _stakeList; + private List _claimStakeList; + private List _migrateMonsterCollectionList; + private List _grindList; + private List _itemEnhancementFailList; + private List _unlockEquipmentRecipeList; + private List _unlockWorldList; + private List _replaceCombinationEquipmentMaterialList; + private List _hasRandomBuffList; + private List _joinArenaList; + private List _battleArenaList; + private List _raiderList; + private List _battleGrandFinaleList; + private List _eventMaterialItemCraftsList; + private List _runeEnhancementList; + private List _unlockRuneSlotList; + private List _rapidCombinationList; + private List _petEnhancementList; + private List _transferAssetList; + private List _requestPledgeList; + + [Command(Description = "Migrate action data in rocksdb store to mysql db.")] + public void Migration( + [Option('o', Description = "Rocksdb path to migrate.")] + string storePath, + [Option( + "mysql-server", + Description = "A hostname of MySQL server.")] + string mysqlServer, + [Option( + "mysql-port", + Description = "A port of MySQL server.")] + uint mysqlPort, + [Option( + "mysql-username", + Description = "The name of MySQL user.")] + string mysqlUsername, + [Option( + "mysql-password", + Description = "The password of MySQL user.")] + string mysqlPassword, + [Option( + "mysql-database", + Description = "The name of MySQL database to use.")] + string mysqlDatabase, + [Option( + "offset", + Description = "offset of block index (no entry will migrate from the genesis block).")] + int? offset = null, + [Option( + "limit", + Description = "limit of block count (no entry will migrate to the chain tip).")] + int? limit = null + ) + { + DateTimeOffset start = DateTimeOffset.UtcNow; + var builder = new MySqlConnectionStringBuilder + { + Database = mysqlDatabase, + UserID = mysqlUsername, + Password = mysqlPassword, + Server = mysqlServer, + Port = mysqlPort, + AllowLoadLocalInfile = true, + }; + + _connectionString = builder.ConnectionString; + var dbContextOptions = + new DbContextOptionsBuilder() + .UseMySql(_connectionString, ServerVersion.AutoDetect(_connectionString)).Options; + var serviceCollection = new ServiceCollection(); + IServiceProvider provider = serviceCollection.BuildServiceProvider(); + IDbContextFactory dbContextFactory = new DbContextFactory( + provider, + dbContextOptions, + new DbContextFactorySource()); + _mySqlStore = new MySqlStore(dbContextFactory); + + Console.WriteLine("Setting up RocksDBStore..."); + _baseStore = new RocksDBStore( + storePath, + dbConnectionCacheSize: 10000); + long totalLength = _baseStore.CountBlocks(); + + if (totalLength == 0) + { + throw new CommandExitedException("Invalid rocksdb-store. Please enter a valid store path", -1); + } + + if (!(_baseStore.GetCanonicalChainId() is { } chainId)) + { + Console.Error.WriteLine("There is no canonical chain: {0}", storePath); + Environment.Exit(1); + return; + } + + if (!(_baseStore.IndexBlockHash(chainId, 0) is { } gHash)) + { + Console.Error.WriteLine("There is no genesis block: {0}", storePath); + Environment.Exit(1); + return; + } + + // Setup base store + RocksDBKeyValueStore baseStateKeyValueStore = new RocksDBKeyValueStore(Path.Combine(storePath, "states")); + TrieStateStore baseStateStore = + new TrieStateStore(baseStateKeyValueStore); + + // Setup block policy + IStagePolicy stagePolicy = new VolatileStagePolicy(); + var blockPolicySource = new BlockPolicySource(); + IBlockPolicy blockPolicy = blockPolicySource.GetPolicy(); + + // Setup base chain & new chain + Block genesis = _baseStore.GetBlock(gHash); + var blockChainStates = new BlockChainStates(_baseStore, baseStateStore); + var actionEvaluator = new ActionEvaluator( + _ => blockPolicy.BlockAction, + blockChainStates, + new NCActionLoader()); + _baseChain = new BlockChain(blockPolicy, stagePolicy, _baseStore, baseStateStore, genesis, blockChainStates, actionEvaluator); + + // Check offset and limit value based on chain height + long height = _baseChain.Tip.Index; + if (offset + limit > (int)height) + { + Console.Error.WriteLine( + "The sum of the offset and limit is greater than the chain tip index: {0}", + height); + Environment.Exit(1); + return; + } + + Console.WriteLine("Start migration."); + + // lists to keep track of inserted addresses to minimize duplicates + _agentCheck = new List(); + _avatarCheck = new List(); + + _blockList = new List(); + _txList = new List(); + _agentList = new List(); + _avatarList = new List(); + _hackAndSlashList = new List(); + _hasWithRandomBuffList = new List(); + _claimStakeRewardList = new List(); + _runesAcquiredList = new List(); + _eventDungeonBattleList = new List(); + _eventConsumableItemCraftsList = new List(); + _hackAndSlashSweepList = new List(); + _combinationConsumableList = new List(); + _combinationEquipmentList = new List(); + _equipmentList = new List(); + _itemEnhancementList = new List(); + _buyShopEquipmentsList = new List(); + _buyShopCostumesList = new List(); + _buyShopMaterialsList = new List(); + _buyShopConsumablesList = new List(); + _stakeList = new List(); + _claimStakeList = new List(); + _migrateMonsterCollectionList = new List(); + _grindList = new List(); + _itemEnhancementFailList = new List(); + _unlockEquipmentRecipeList = new List(); + _unlockWorldList = new List(); + _replaceCombinationEquipmentMaterialList = new List(); + _hasRandomBuffList = new List(); + _joinArenaList = new List(); + _battleArenaList = new List(); + _raiderList = new List(); + _battleGrandFinaleList = new List(); + _eventMaterialItemCraftsList = new List(); + _runeEnhancementList = new List(); + _unlockRuneSlotList = new List(); + _rapidCombinationList = new List(); + _petEnhancementList = new List(); + _transferAssetList = new List(); + _requestPledgeList = new List(); + + try + { + int totalCount = limit ?? (int)_baseStore.CountBlocks(); + int remainingCount = totalCount; + int offsetIdx = 0; + + while (remainingCount > 0) + { + int interval = 100; + int limitInterval; + Task>[] taskArray; + if (interval < remainingCount) + { + taskArray = new Task>[interval]; + limitInterval = interval; + } + else + { + taskArray = new Task>[remainingCount]; + limitInterval = remainingCount; + } + + foreach (var item in + _baseStore.IterateIndexes(_baseChain.Id, offset + offsetIdx ?? 0 + offsetIdx, limitInterval).Select((value, i) => new { i, value })) + { + var block = _baseStore.GetBlock(item.value); + _blockList.Add(BlockData.GetBlockInfo(block)); + _blockHash = block.Hash; + _blockIndex = block.Index; + _blockTimeOffset = block.Timestamp; + foreach (var tx in block.Transactions) + { + _txList.Add(TransactionData.GetTransactionInfo(block, tx)); + + // check if address is already in _agentCheck + if (!_agentCheck.Contains(tx.Signer.ToString())) + { + _agentList.Add(AgentData.GetAgentInfo(tx.Signer)); + _agentCheck.Add(tx.Signer.ToString()); + } + } + + taskArray[item.i] = Task.Factory.StartNew(() => + { + List actionEvaluations = EvaluateBlock(block); + Console.WriteLine($"Block progress: #{block.Index}/{remainingCount}"); + return actionEvaluations; + }); + } + + if (interval < remainingCount) + { + remainingCount -= interval; + offsetIdx += interval; + } + else + { + remainingCount = 0; + offsetIdx += remainingCount; + } + + Task.WaitAll(taskArray); + ProcessTasks(taskArray); + } + + DateTimeOffset postDataPrep = _blockTimeOffset; + Console.WriteLine("Data Preparation Complete! Time Elapsed: {0}", postDataPrep - start); + Console.WriteLine("Start Data Migration..."); + _mySqlStore.StoreItemEnhancementList(_itemEnhancementList); + } + catch (Exception e) + { + Console.WriteLine(e.Message); + } + + DateTimeOffset end = DateTimeOffset.UtcNow; + Console.WriteLine("Migration Complete! Time Elapsed: {0}", end - start); + } + + private void ProcessTasks(Task>[] taskArray) + { + foreach (var task in taskArray) + { + if (task.Result is { } data) + { + foreach (var ae in data) + { + var actionLoader = new NCActionLoader(); + if (actionLoader.LoadAction(_blockIndex, ae.Action) is ActionBase action && action is ItemEnhancement itemEnhancement) + { + var start = DateTimeOffset.UtcNow; + _itemEnhancementList.Add(ItemEnhancementData.GetItemEnhancementInfo( + ae.InputContext.PreviousState, + ae.OutputState, + ae.InputContext.Signer, + itemEnhancement.avatarAddress, + itemEnhancement.slotIndex, + Guid.Empty, + itemEnhancement.materialIds, + itemEnhancement.itemId, + itemEnhancement.Id, + ae.InputContext.BlockIndex)); + var end = DateTimeOffset.UtcNow; + Console.WriteLine("Writing ItemEnhancement action in block #{0}. Time Taken: {1} ms.", ae.InputContext.BlockIndex, (end - start).Milliseconds); + } + } + } + } + } + + private List EvaluateBlock(Block block) + { + var evList = _baseChain.EvaluateBlock(block).ToList(); + return evList; + } + } +} diff --git a/NineChronicles.DataProvider.Executable/Program.cs b/NineChronicles.DataProvider.Executable/Program.cs index f36e2772..147eca15 100644 --- a/NineChronicles.DataProvider.Executable/Program.cs +++ b/NineChronicles.DataProvider.Executable/Program.cs @@ -26,6 +26,7 @@ namespace NineChronicles.DataProvider.Executable [HasSubCommands(typeof(MySqlMigration), "mysql-migration")] [HasSubCommands(typeof(BattleArenaRankingMigration), "battle-arena-ranking-migration")] [HasSubCommands(typeof(UserStakingMigration), "user-staking-migration")] + [HasSubCommands(typeof(ItemEnhancementMigration), "item-enhancement-migration")] public class Program : CoconaLiteConsoleAppBase { public static async Task Main(string[] args)