From 8985dfea0d11a4e552994562ef267b8def7d0a19 Mon Sep 17 00:00:00 2001 From: alexandr Date: Thu, 17 May 2018 13:08:52 +0300 Subject: [PATCH 01/52] NODE-129: Rollback genesis.it.conf, constants moved from PoSCalculator, simplified normalizeBaseTarget --- .../consensus/PoSCalculator.scala | 22 +++++++--------- src/test/resources/genesis.dev.conf | 21 +++++++++++++++ src/test/resources/genesis.it.conf | 26 +++++++++++-------- .../consensus/FairPoSCalculatorTest.scala | 2 +- 4 files changed, 47 insertions(+), 24 deletions(-) create mode 100644 src/test/resources/genesis.dev.conf diff --git a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala index 0742ef67e26..5e438473d5e 100644 --- a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala +++ b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala @@ -6,17 +6,9 @@ import com.wavesplatform.features.FeatureProvider._ import com.wavesplatform.state.Blockchain trait PoSCalculator { - protected val HitSize: Int = 8 - protected val MeanCalculationDepth: Int = 3 - - // Min BaseTarget value is 9 because only in this case it is possible to get to next integer value (10) - // then increasing base target by 11% and casting it to Long afterward (see lines 55 and 59) + protected val HitSize: Int = 8 protected val MinBaseTarget: Long = 9 - protected val MinBlockDelaySeconds = 53 - protected val MaxBlockDelaySeconds = 67 - protected val BaseTargetGamma = 64 - def generatorSignature(signature: Array[Byte], publicKey: Array[Byte]): Array[Byte] = { val s = new Array[Byte](crypto.DigestSize * 2) System.arraycopy(signature, 0, s, 0, crypto.DigestSize) @@ -47,8 +39,9 @@ trait PoSCalculator { value * targetBlockDelaySeconds / (60: Double) protected def normalizeBaseTarget(baseTarget: Long, targetBlockDelaySeconds: Long): Long = { - val maxBaseTarget = Long.MaxValue / targetBlockDelaySeconds - if (baseTarget < MinBaseTarget) MinBaseTarget else if (baseTarget > maxBaseTarget) maxBaseTarget else baseTarget + baseTarget + .max(MinBaseTarget) + .min(Long.MaxValue / targetBlockDelaySeconds) } } @@ -68,13 +61,18 @@ class PoSSelector(val blockchain: Blockchain) extends PoSCalculator { pos.baseTarget(targetBlockDelaySeconds, prevHeight, prevBaseTarget, parentTimestamp, maybeGreatGrandParentTimestamp, timestamp) } - def calculateDelay(hit: BigInt, bt: Long, balance: Long) = + def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long = pos.calculateDelay(hit, bt, balance) private def fair(height: Int): Boolean = blockchain.activatedFeaturesAt(height).contains(BlockchainFeatures.FairPoS.id) } object NxtPoSCalculator extends PoSCalculator { + protected val MinBlockDelaySeconds = 53 + protected val MaxBlockDelaySeconds = 67 + protected val BaseTargetGamma = 64 + protected val MeanCalculationDepth = 3 + def baseTarget(targetBlockDelaySeconds: Long, prevHeight: Int, prevBaseTarget: Long, diff --git a/src/test/resources/genesis.dev.conf b/src/test/resources/genesis.dev.conf new file mode 100644 index 00000000000..2f7af5b1f56 --- /dev/null +++ b/src/test/resources/genesis.dev.conf @@ -0,0 +1,21 @@ +# Configuration for genesis block generator +# To generate run from SBT: test:run src/test/resources/genesis.it.conf +genesis-generator { + network-type: "D" + + initial-balance: 10000000000000000 + base-target: 100 + average-block-delay: 60s + timestamp: 1489352400000 # Comment to use the current time + + # seed text -> share + # the sum of shares should be <= initial-balance + distributions { + "create genesis wallet devnet-0": 200000000000000 + "create genesis wallet devnet-0-d": 500000000000000 + "create genesis wallet devnet-1": 1000000000000000 + "create genesis wallet devnet-1-d": 1500000000000000 + "create genesis wallet devnet-2": 2000000000000000 + "create genesis wallet devnet-2-d": 2500000000000000 + } +} diff --git a/src/test/resources/genesis.it.conf b/src/test/resources/genesis.it.conf index 2f7af5b1f56..b0b2368dd2c 100644 --- a/src/test/resources/genesis.it.conf +++ b/src/test/resources/genesis.it.conf @@ -1,21 +1,25 @@ # Configuration for genesis block generator # To generate run from SBT: test:run src/test/resources/genesis.it.conf genesis-generator { - network-type: "D" + network-type: "I" - initial-balance: 10000000000000000 - base-target: 100 - average-block-delay: 60s + initial-balance: 1000000000000000 + base-target: 200000 + average-block-delay: 10s timestamp: 1489352400000 # Comment to use the current time # seed text -> share # the sum of shares should be <= initial-balance distributions { - "create genesis wallet devnet-0": 200000000000000 - "create genesis wallet devnet-0-d": 500000000000000 - "create genesis wallet devnet-1": 1000000000000000 - "create genesis wallet devnet-1-d": 1500000000000000 - "create genesis wallet devnet-2": 2000000000000000 - "create genesis wallet devnet-2-d": 2500000000000000 + "node01": 1400000000000 + "node02": 1500000000000 + "node03": 1600000000000 + "node04": 4000000000000 + "node05": 5000000000000 + "node06": 6000000000000 + "node07": 40000000000000 + "node08": 50000000000000 + "node09": 60000000000000 + "node10": 830500000000000 } -} +} \ No newline at end of file diff --git a/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala b/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala index f9c82be65b7..35c63e8e680 100644 --- a/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala +++ b/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala @@ -15,7 +15,7 @@ class FairPoSCalculatorTest extends PropSpec with Matchers { arr } - property("") { + property("Correct consensus parameters of blocks generated with FairPoS") { val balance = 50000000L * 100000000L From a05f001998f3adfd7f5148083237c825e1b0851e Mon Sep 17 00:00:00 2001 From: Dmitry Shuranov Date: Thu, 17 May 2018 16:20:49 +0300 Subject: [PATCH 02/52] Increased max timestamp diff for orders request from 3 hours to 30 days --- src/main/resources/application.conf | 2 +- .../wavesplatform/settings/MatcherSettingsSpecification.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 1360803f4b3..f6bbc9d95ed 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -279,7 +279,7 @@ waves { predefined-pairs: [] # Maximum difference with Matcher server time - max-timestamp-diff = 3h + max-timestamp-diff = 30d # Blacklisted assets id blacklisted-assets: [] diff --git a/src/test/scala/com/wavesplatform/settings/MatcherSettingsSpecification.scala b/src/test/scala/com/wavesplatform/settings/MatcherSettingsSpecification.scala index 5d33d5b96f2..cd12b4512b4 100644 --- a/src/test/scala/com/wavesplatform/settings/MatcherSettingsSpecification.scala +++ b/src/test/scala/com/wavesplatform/settings/MatcherSettingsSpecification.scala @@ -34,7 +34,7 @@ class MatcherSettingsSpecification extends FlatSpec with Matchers { | {amountAsset = "DHgwrRvVyqJsepd32YbBqUeDH4GJ1N984X8QoekjgH8J", priceAsset = "WAVES"}, | {amountAsset = "DHgwrRvVyqJsepd32YbBqUeDH4GJ1N984X8QoekjgH8J", priceAsset = "8LQW8f7P5d5PZM7GtZEBgaqRPGSzS3DfPuiXrURJ4AJS"}, | ] - | max-timestamp-diff = 3h + | max-timestamp-diff = 30d | blacklisted-assets: ["a"] | blacklisted-names: ["b"] | blacklisted-addresses: ["c"] From 466da3cd378f51039bc158d98485a8173c6ece10 Mon Sep 17 00:00:00 2001 From: Sergey Nazarov Date: Thu, 17 May 2018 17:48:17 +0300 Subject: [PATCH 03/52] Revert "Node 129 fair proof of stake" (cherry picked from commit 7ac840b7b8c42fd71f0b6435dcd7ec5093790003) --- .../wavesplatform/it/BaseTargetChecker.scala | 19 +-- .../wavesplatform/it/api/SyncHttpApi.scala | 6 - .../it/sync/FairPoSTestSuite.scala | 49 ------ .../scala/com/wavesplatform/Application.scala | 11 +- .../scala/com/wavesplatform/Importer.scala | 4 +- .../consensus/GeneratingBalanceProvider.scala | 30 ---- .../consensus/PoSCalculator.scala | 138 ---------------- .../features/BlockchainFeature.scala | 4 +- .../com/wavesplatform/mining/Miner.scala | 151 +++++++----------- .../state/appender/BlockAppender.scala | 7 +- .../state/appender/ExtensionAppender.scala | 9 +- .../state/appender/package.scala | 46 +++--- .../scorex/api/http/AddressApiRoute.scala | 5 +- .../nxt/api/http/NxtConsensusApiRoute.scala | 5 +- .../scala/scorex/transaction/PoSCalc.scala | 109 +++++++++++++ src/test/resources/genesis.dev.conf | 21 --- src/test/resources/genesis.it.conf | 2 +- .../consensus/FairPoSCalculatorTest.scala | 72 --------- waves-devnet.conf | 16 +- 19 files changed, 226 insertions(+), 478 deletions(-) delete mode 100644 it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala delete mode 100644 src/main/scala/com/wavesplatform/consensus/GeneratingBalanceProvider.scala delete mode 100644 src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala create mode 100644 src/main/scala/scorex/transaction/PoSCalc.scala delete mode 100644 src/test/resources/genesis.dev.conf delete mode 100644 src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala diff --git a/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala b/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala index 77b51a5904c..39b64e27c05 100644 --- a/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala +++ b/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala @@ -3,7 +3,6 @@ package com.wavesplatform.it import java.time.Instant import com.typesafe.config.ConfigFactory.{defaultApplication, defaultReference} -import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.db.openDB import com.wavesplatform.history.StorageFactory import com.wavesplatform.settings._ @@ -11,34 +10,36 @@ import com.wavesplatform.state.{ByteStr, EitherExt2} import net.ceedubs.ficus.Ficus._ import scorex.account.PublicKeyAccount import scorex.block.Block +import scorex.transaction.PoSCalc import scorex.utils.NTP object BaseTargetChecker { def main(args: Array[String]): Unit = { - val docker = Docker(getClass) + val startTs = System.currentTimeMillis() + val docker = Docker(getClass) val sharedConfig = docker.genesisOverride .withFallback(docker.configTemplate) .withFallback(defaultApplication()) .withFallback(defaultReference()) .resolve() val settings = WavesSettings.fromConfig(sharedConfig) + val fs = settings.blockchainSettings.functionalitySettings val genesisBlock = Block.genesis(settings.blockchainSettings.genesisSettings).explicitGet() val db = openDB("/tmp/tmp-db", 1024) val bu = StorageFactory(settings, db, NTP) - val pos = new PoSSelector(bu) bu.processBlock(genesisBlock) println(s"Genesis TS = ${Instant.ofEpochMilli(genesisBlock.timestamp)}") val m = NodeConfigs.Default.map(_.withFallback(sharedConfig)).collect { case cfg if cfg.as[Boolean]("waves.miner.enable") => - val account = PublicKeyAccount(cfg.as[ByteStr]("public-key").arr) - val address = account.toAddress - val balance = bu.balance(address, None) - val consensus = genesisBlock.consensusData - val timeDelay = pos.validBlockDelay(consensus.generationSignature.arr, account.publicKey, consensus.baseTarget, balance) + val publicKey = PublicKeyAccount(cfg.as[ByteStr]("public-key").arr) + val address = publicKey.toAddress + PoSCalc.nextBlockGenerationTime(1, bu, fs, genesisBlock, publicKey) match { + case Right((_, ts)) => f"$address: ${(ts - startTs) * 1e-3}%10.3f s" + case _ => s"$address: n/a" + } - f"$address: ${timeDelay * 1e-3}%10.3f s" } docker.close() diff --git a/it/src/main/scala/com/wavesplatform/it/api/SyncHttpApi.scala b/it/src/main/scala/com/wavesplatform/it/api/SyncHttpApi.scala index 0fac216a7e3..01c068912e3 100644 --- a/it/src/main/scala/com/wavesplatform/it/api/SyncHttpApi.scala +++ b/it/src/main/scala/com/wavesplatform/it/api/SyncHttpApi.scala @@ -61,9 +61,6 @@ object SyncHttpApi extends Assertions { def postJsonWithApiKey[A: Writes](path: String, body: A): Response = Await.result(async(n).postJsonWithApiKey(path, body), RequestAwaitTime) - def accountBalance(acc: String): Long = - Await.result(async(n).accountBalance(acc), RequestAwaitTime) - def accountBalances(acc: String): (Long, Long) = Await.result(async(n).accountBalances(acc), RequestAwaitTime) @@ -167,9 +164,6 @@ object SyncHttpApi extends Assertions { def height: Int = Await.result(async(n).height, RequestAwaitTime) - - def blockHeadersSeq(from: Int, to: Int): Seq[BlockHeaders] = - Await.result(async(n).blockHeadersSeq(from, to), RequestAwaitTime) } implicit class NodesExtSync(nodes: Seq[Node]) { diff --git a/it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala deleted file mode 100644 index c8dc7e88c4a..00000000000 --- a/it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.wavesplatform.it.sync - -import com.typesafe.config.{Config, ConfigFactory} -import org.scalatest.{CancelAfterFailure, FunSuite} -import com.wavesplatform.it.api.State -import com.wavesplatform.it.api.SyncHttpApi._ -import com.wavesplatform.it.transactions.NodesFromDocker -import com.wavesplatform.it.util._ -import scala.concurrent.duration._ - -class FairPoSTestSuite extends FunSuite with CancelAfterFailure with NodesFromDocker { - import FairPoSTestSuite._ - - override protected def nodeConfigs: Seq[Config] = Configs - - private val transferFee = 0.001.waves - private val transferAmount = 1000.waves - - test("blockchain grows with FairPoS activated") { - nodes.head.waitForHeight(10, 3.minutes) - - val txId = nodes.head.transfer(nodes.head.address, nodes.last.address, transferAmount, transferFee).id - nodes.last.waitForTransaction(txId) - - val heightAfterTransfer = nodes.head.height - - nodes.head.waitForHeight(heightAfterTransfer + 20, 5.minutes) - } -} - -object FairPoSTestSuite { - import com.wavesplatform.it.NodeConfigs._ - private val microblockActivationHeight = 0 - private val fairPoSActivationHeight = 10 - - private val config = - ConfigFactory.parseString(s""" - |waves { - | blockchain.custom { - | functionality { - | pre-activated-features {1 = $microblockActivationHeight, 8 = $fairPoSActivationHeight} - | generation-balance-depth-from-50-to-1000-after-height = 1000 - | } - | } - | miner.quorum = 1 - |}""".stripMargin) - - val Configs: Seq[Config] = Default.map(config.withFallback(_)).take(4) -} diff --git a/src/main/scala/com/wavesplatform/Application.scala b/src/main/scala/com/wavesplatform/Application.scala index 0437750a8de..c462d84da49 100644 --- a/src/main/scala/com/wavesplatform/Application.scala +++ b/src/main/scala/com/wavesplatform/Application.scala @@ -11,7 +11,6 @@ import akka.stream.ActorMaterializer import cats.instances.all._ import com.typesafe.config._ import com.wavesplatform.actor.RootActorSystem -import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.db.openDB import com.wavesplatform.features.api.ActivationApiRoute import com.wavesplatform.history.{CheckpointServiceImpl, StorageFactory} @@ -124,25 +123,19 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con maybeUtx = Some(utxStorage) val knownInvalidBlocks = new InvalidBlockStorageImpl(settings.synchronizationSettings.invalidBlocksStorage) - - val pos = new PoSSelector(blockchainUpdater) - val miner = if (settings.minerSettings.enable) - new MinerImpl(allChannels, blockchainUpdater, checkpointService, settings, time, utxStorage, wallet, pos, minerScheduler, appenderScheduler) + new MinerImpl(allChannels, blockchainUpdater, checkpointService, settings, time, utxStorage, wallet, minerScheduler, appenderScheduler) else Miner.Disabled val processBlock = - BlockAppender(checkpointService, blockchainUpdater, time, utxStorage, pos, settings, allChannels, peerDatabase, miner, appenderScheduler) _ - + BlockAppender(checkpointService, blockchainUpdater, time, utxStorage, settings, allChannels, peerDatabase, miner, appenderScheduler) _ val processCheckpoint = CheckpointAppender(checkpointService, blockchainUpdater, blockchainUpdater, peerDatabase, miner, allChannels, appenderScheduler) _ - val processFork = ExtensionAppender( checkpointService, blockchainUpdater, utxStorage, - pos, time, settings, knownInvalidBlocks, diff --git a/src/main/scala/com/wavesplatform/Importer.scala b/src/main/scala/com/wavesplatform/Importer.scala index f791c87c1fc..b9c8f4aa08a 100644 --- a/src/main/scala/com/wavesplatform/Importer.scala +++ b/src/main/scala/com/wavesplatform/Importer.scala @@ -4,7 +4,6 @@ import java.io._ import com.google.common.primitives.Ints import com.typesafe.config.ConfigFactory -import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.db.openDB import com.wavesplatform.history.{CheckpointServiceImpl, StorageFactory} import com.wavesplatform.mining.MultiDimensionalMiningConstraint @@ -58,9 +57,8 @@ object Importer extends ScorexLogging { case Success(inputStream) => val db = openDB(settings.dataDirectory, settings.levelDbCacheSize) val blockchainUpdater = StorageFactory(settings, db, NTP) - val pos = new PoSSelector(blockchainUpdater) val checkpoint = new CheckpointServiceImpl(db, settings.checkpointsSettings) - val extAppender = BlockAppender(checkpoint, blockchainUpdater, NTP, utxPoolStub, pos, settings, scheduler) _ + val extAppender = BlockAppender(checkpoint, blockchainUpdater, NTP, utxPoolStub, settings, scheduler) _ checkGenesis(settings, blockchainUpdater) val bis = new BufferedInputStream(inputStream) var quit = false diff --git a/src/main/scala/com/wavesplatform/consensus/GeneratingBalanceProvider.scala b/src/main/scala/com/wavesplatform/consensus/GeneratingBalanceProvider.scala deleted file mode 100644 index 39bf5aebc28..00000000000 --- a/src/main/scala/com/wavesplatform/consensus/GeneratingBalanceProvider.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.wavesplatform.consensus - -import com.wavesplatform.features.BlockchainFeatures -import com.wavesplatform.settings.FunctionalitySettings -import com.wavesplatform.state.Blockchain -import scorex.account.Address -import scorex.block.Block - -object GeneratingBalanceProvider { - private val MinimalEffectiveBalanceForGenerator1: Long = 1000000000000L - private val MinimalEffectiveBalanceForGenerator2: Long = 100000000000L - private val FirstDepth = 50 - private val SecondDepth = 1000 - - def isMiningAllowed(blockchain: Blockchain, height: Int, effectiveBalance: Long): Boolean = { - val activated = blockchain.activatedFeatures.get(BlockchainFeatures.SmallerMinimalGeneratingBalance.id).exists(height >= _) - (!activated && effectiveBalance >= MinimalEffectiveBalanceForGenerator1) || (activated && effectiveBalance >= MinimalEffectiveBalanceForGenerator2) - } - - def isEffectiveBalanceValid(blockchain: Blockchain, fs: FunctionalitySettings, height: Int, block: Block, effectiveBalance: Long): Boolean = - block.timestamp < fs.minimalGeneratingBalanceAfter || (block.timestamp >= fs.minimalGeneratingBalanceAfter && effectiveBalance >= MinimalEffectiveBalanceForGenerator1) || - blockchain.activatedFeatures - .get(BlockchainFeatures.SmallerMinimalGeneratingBalance.id) - .exists(height >= _) && effectiveBalance >= MinimalEffectiveBalanceForGenerator2 - - def balance(blockchain: Blockchain, fs: FunctionalitySettings, height: Int, account: Address): Long = { - val depth = if (height >= fs.generationBalanceDepthFrom50To1000AfterHeight) SecondDepth else FirstDepth - blockchain.effectiveBalance(account, height, depth) - } -} diff --git a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala deleted file mode 100644 index 5e438473d5e..00000000000 --- a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.wavesplatform.consensus - -import com.wavesplatform.crypto -import com.wavesplatform.features.BlockchainFeatures -import com.wavesplatform.features.FeatureProvider._ -import com.wavesplatform.state.Blockchain - -trait PoSCalculator { - protected val HitSize: Int = 8 - protected val MinBaseTarget: Long = 9 - - def generatorSignature(signature: Array[Byte], publicKey: Array[Byte]): Array[Byte] = { - val s = new Array[Byte](crypto.DigestSize * 2) - System.arraycopy(signature, 0, s, 0, crypto.DigestSize) - System.arraycopy(publicKey, 0, s, crypto.DigestSize, crypto.DigestSize) - crypto.fastHash(s) - } - - def baseTarget(targetBlockDelaySeconds: Long, - prevHeight: Int, - prevBaseTarget: Long, - parentTimestamp: Long, - maybeGreatGrandParentTimestamp: Option[Long], - timestamp: Long): Long - - protected def hit(generatorSignature: Array[Byte]): BigInt = BigInt(1, generatorSignature.take(HitSize).reverse) - - def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long - - def validBlockDelay(genSig: Array[Byte], baseTarget: Long, balance: Long): Long = { - calculateDelay(hit(genSig), baseTarget, balance) - } - - def validBlockDelay(genSig: Array[Byte], publicKey: Array[Byte], baseTarget: Long, balance: Long): Long = { - calculateDelay(hit(generatorSignature(genSig, publicKey)), baseTarget, balance) - } - - protected def normalize(value: Long, targetBlockDelaySeconds: Long): Double = - value * targetBlockDelaySeconds / (60: Double) - - protected def normalizeBaseTarget(baseTarget: Long, targetBlockDelaySeconds: Long): Long = { - baseTarget - .max(MinBaseTarget) - .min(Long.MaxValue / targetBlockDelaySeconds) - } - -} - -class PoSSelector(val blockchain: Blockchain) extends PoSCalculator { - - protected def pos: PoSCalculator = - if (fair(blockchain.height)) FairPoSCalculator - else NxtPoSCalculator - - override def baseTarget(targetBlockDelaySeconds: Long, - prevHeight: Int, - prevBaseTarget: Long, - parentTimestamp: Long, - maybeGreatGrandParentTimestamp: Option[Long], - timestamp: Long): Long = { - pos.baseTarget(targetBlockDelaySeconds, prevHeight, prevBaseTarget, parentTimestamp, maybeGreatGrandParentTimestamp, timestamp) - } - - def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long = - pos.calculateDelay(hit, bt, balance) - - private def fair(height: Int): Boolean = blockchain.activatedFeaturesAt(height).contains(BlockchainFeatures.FairPoS.id) -} - -object NxtPoSCalculator extends PoSCalculator { - protected val MinBlockDelaySeconds = 53 - protected val MaxBlockDelaySeconds = 67 - protected val BaseTargetGamma = 64 - protected val MeanCalculationDepth = 3 - - def baseTarget(targetBlockDelaySeconds: Long, - prevHeight: Int, - prevBaseTarget: Long, - parentTimestamp: Long, - maybeGreatGrandParentTimestamp: Option[Long], - timestamp: Long): Long = { - - if (prevHeight % 2 == 0) { - val meanBlockDelay = maybeGreatGrandParentTimestamp.fold(timestamp - parentTimestamp)(ts => (timestamp - ts) / MeanCalculationDepth) / 1000 - val minBlockDelay = normalize(MinBlockDelaySeconds, targetBlockDelaySeconds) - val maxBlockDelay = normalize(MaxBlockDelaySeconds, targetBlockDelaySeconds) - val baseTargetGamma = normalize(BaseTargetGamma, targetBlockDelaySeconds) - - val baseTarget = (if (meanBlockDelay > targetBlockDelaySeconds) { - prevBaseTarget * Math.min(meanBlockDelay, maxBlockDelay) / targetBlockDelaySeconds - } else { - prevBaseTarget - prevBaseTarget * baseTargetGamma * - (targetBlockDelaySeconds - Math.max(meanBlockDelay, minBlockDelay)) / (targetBlockDelaySeconds * 100) - }).toLong - - normalizeBaseTarget(baseTarget, targetBlockDelaySeconds) - } else { - prevBaseTarget - } - } - - def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long = ((hit * 1000) / (BigInt(bt) * balance)).toLong - -} - -object FairPoSCalculator extends PoSCalculator { - private val MaxSignature: Array[Byte] = Array.fill[Byte](HitSize)(-1) - private val MaxHit: BigDecimal = BigDecimal(BigInt(1, MaxSignature)) - private val C1 = 70000 - private val C2 = 5E17 - private val TMin = 5000 - - def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long = { - val h = (BigDecimal(hit) / MaxHit).toDouble - val a = TMin + C1 * math.log(1 - C2 * math.log(h) / bt / balance) - a.toLong - } - - def baseTarget(targetBlockDelaySeconds: Long, - prevHeight: Int, - prevBaseTarget: Long, - parentTimestamp: Long, - maybeGreatGrandParentTimestamp: Option[Long], - timestamp: Long): Long = { - val maxDelay = normalize(90, targetBlockDelaySeconds) - val minDelay = normalize(30, targetBlockDelaySeconds) - - maybeGreatGrandParentTimestamp match { - case None => - prevBaseTarget - case Some(ts) => - val avg = (timestamp - ts) / 3 / 1000 - if (avg > maxDelay) prevBaseTarget + math.max(1, prevBaseTarget / 100) - else if (avg < minDelay) prevBaseTarget - math.max(1, prevBaseTarget / 100) - else prevBaseTarget - } - } -} diff --git a/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala b/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala index c2215b53d4a..8c9d3e31f8a 100644 --- a/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala +++ b/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala @@ -11,7 +11,6 @@ object BlockchainFeatures { val DataTransaction = BlockchainFeature(5, "Data Transaction") val BurnAnyTokens = BlockchainFeature(6, "Burn Any Tokens") val FeeSponsorship = BlockchainFeature(7, "Fee Sponsorship") - val FairPoS = BlockchainFeature(8, "Fair PoS") private val dict = Seq( SmallerMinimalGeneratingBalance, @@ -20,8 +19,7 @@ object BlockchainFeatures { SmartAccounts, DataTransaction, BurnAnyTokens, - FeeSponsorship, - FairPoS + FeeSponsorship ).map(f => f.id -> f).toMap val implemented: Set[Short] = dict.keySet diff --git a/src/main/scala/com/wavesplatform/mining/Miner.scala b/src/main/scala/com/wavesplatform/mining/Miner.scala index 35cf37a137a..09225a5fb6c 100644 --- a/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -1,12 +1,11 @@ package com.wavesplatform.mining import cats.data.EitherT -import com.wavesplatform.consensus.{GeneratingBalanceProvider, PoSCalculator} import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.features.FeatureProvider._ import com.wavesplatform.metrics.{BlockStats, HistogramExt, Instrumented} import com.wavesplatform.network._ -import com.wavesplatform.settings.{FunctionalitySettings, WavesSettings} +import com.wavesplatform.settings.WavesSettings import com.wavesplatform.state._ import com.wavesplatform.state.appender.{BlockAppender, MicroblockAppender} import com.wavesplatform.utx.UtxPool @@ -16,10 +15,11 @@ import kamon.metric.instrument import monix.eval.Task import monix.execution.cancelables.{CompositeCancelable, SerialCancelable} import monix.execution.schedulers.SchedulerService -import scorex.account.{Address, PrivateKeyAccount, PublicKeyAccount} +import scorex.account.{Address, PrivateKeyAccount} import scorex.block.Block._ import scorex.block.{Block, MicroBlock} import scorex.consensus.nxt.NxtLikeConsensusBlockData +import scorex.transaction.PoSCalc._ import scorex.transaction._ import scorex.utils.{ScorexLogging, Time} import scorex.wallet.Wallet @@ -59,7 +59,6 @@ class MinerImpl(allChannels: ChannelGroup, timeService: Time, utx: UtxPool, wallet: Wallet, - pos: PoSCalculator, val minerScheduler: SchedulerService, val appenderScheduler: SchedulerService) extends Miner @@ -101,75 +100,60 @@ class MinerImpl(allChannels: ChannelGroup, private def ngEnabled: Boolean = blockchainUpdater.featureActivationHeight(BlockchainFeatures.NG.id).exists(blockchainUpdater.height > _ + 1) private def generateOneBlockTask(account: PrivateKeyAccount, balance: Long)( - delay: FiniteDuration): Task[Either[String, (MiningConstraints, Block, MiningConstraint)]] = { + delay: FiniteDuration): Task[Either[String, (MiningConstraints, Block, MiningConstraint)]] = Task { - forgeBlock(account, balance) - }.delayExecution(delay) - } - - private def forgeBlock(account: PrivateKeyAccount, balance: Long): Either[String, (MiningConstraints, Block, MiningConstraint)] = { - // should take last block right at the time of mining since microblocks might have been added - val height = blockchainUpdater.height - val version = if (height <= blockchainSettings.functionalitySettings.blockVersion3AfterHeight) PlainBlockVersion else NgBlockVersion - val lastBlock = blockchainUpdater.lastBlock.get - val greatGrandParentTimestamp = blockchainUpdater.parent(lastBlock, 2).map(_.timestamp) - val referencedBlockInfo = blockchainUpdater.bestLastBlockInfo(System.currentTimeMillis() - minMicroBlockDurationMills).get - val pc = allChannels.size() - lazy val currentTime = timeService.correctedTime() - lazy val blockDelay = currentTime - lastBlock.timestamp - lazy val validBlockDelay = pos.validBlockDelay( - referencedBlockInfo.consensus.generationSignature.arr, - account.publicKey, - referencedBlockInfo.consensus.baseTarget, - balance - ) - measureSuccessful( - blockBuildTimeStats, - for { - _ <- Either.cond(pc >= minerSettings.quorum, - (), - s"Quorum not available ($pc/${minerSettings.quorum}, not forging block with ${account.address}") - _ <- Either.cond( - blockDelay > validBlockDelay, - (), - s"${System.currentTimeMillis()}: Block delay $blockDelay was NOT less than estimated delay $validBlockDelay, not forging block with ${account.address}" - ) - _ = log.debug( - s"Forging with ${account.address}, Time $blockDelay > Estimated Time $validBlockDelay, balance $balance, prev block ${referencedBlockInfo.blockId}") - _ = log.debug(s"Previous block ID ${referencedBlockInfo.blockId} at $height with target ${referencedBlockInfo.consensus.baseTarget}") - block <- { - val avgBlockDelay = blockchainSettings.genesisSettings.averageBlockDelay - val btg = pos.baseTarget(avgBlockDelay.toSeconds, - height, - referencedBlockInfo.consensus.baseTarget, - referencedBlockInfo.timestamp, - greatGrandParentTimestamp, - currentTime) - val gs = pos.generatorSignature(referencedBlockInfo.consensus.generationSignature.arr, account.publicKey) - val consensusData = NxtLikeConsensusBlockData(btg, ByteStr(gs)) - val sortInBlock = blockchainUpdater.height <= blockchainSettings.functionalitySettings.dontRequireSortedTransactionsAfter - - val estimators = MiningConstraints(minerSettings, blockchainUpdater, height) - val mdConstraint = MultiDimensionalMiningConstraint(estimators.total, estimators.keyBlock) - val (unconfirmed, updatedMdConstraint) = utx.packUnconfirmed(mdConstraint, sortInBlock) - - val features = - if (version <= 2) Set.empty[Short] - else - settings.featuresSettings.supported - .filterNot(blockchainUpdater.approvedFeatures.keySet) - .filter(BlockchainFeatures.implemented) - .toSet - - log.debug(s"Adding ${unconfirmed.size} unconfirmed transaction(s) to new block") - Block.buildAndSign(version.toByte, currentTime, referencedBlockInfo.blockId, consensusData, unconfirmed, account, features) match { - case Left(e) => Left(e.err) - case Right(x) => Right((estimators, x, updatedMdConstraint.constraints.head)) + // should take last block right at the time of mining since microblocks might have been added + val height = blockchainUpdater.height + val version = if (height <= blockchainSettings.functionalitySettings.blockVersion3AfterHeight) PlainBlockVersion else NgBlockVersion + val lastBlock = blockchainUpdater.lastBlock.get + val greatGrandParentTimestamp = blockchainUpdater.parent(lastBlock, 2).map(_.timestamp) + val referencedBlockInfo = blockchainUpdater.bestLastBlockInfo(System.currentTimeMillis() - minMicroBlockDurationMills).get + val pc = allChannels.size() + lazy val currentTime = timeService.correctedTime() + lazy val h = calcHit(referencedBlockInfo.consensus, account) + lazy val t = calcTarget(referencedBlockInfo.timestamp, referencedBlockInfo.consensus.baseTarget, currentTime, balance) + measureSuccessful( + blockBuildTimeStats, + for { + _ <- Either.cond(pc >= minerSettings.quorum, + (), + s"Quorum not available ($pc/${minerSettings.quorum}, not forging block with ${account.address}") + _ <- Either.cond(h < t, (), s"${System.currentTimeMillis()}: Hit $h was NOT less than target $t, not forging block with ${account.address}") + _ = log.debug(s"Forging with ${account.address}, H $h < T $t, balance $balance, prev block ${referencedBlockInfo.blockId}") + _ = log.debug(s"Previous block ID ${referencedBlockInfo.blockId} at $height with target ${referencedBlockInfo.consensus.baseTarget}") + block <- { + val avgBlockDelay = blockchainSettings.genesisSettings.averageBlockDelay + val btg = calcBaseTarget(avgBlockDelay, + height, + referencedBlockInfo.consensus.baseTarget, + referencedBlockInfo.timestamp, + greatGrandParentTimestamp, + currentTime) + val gs = calcGeneratorSignature(referencedBlockInfo.consensus, account) + val consensusData = NxtLikeConsensusBlockData(btg, ByteStr(gs)) + val sortInBlock = blockchainUpdater.height <= blockchainSettings.functionalitySettings.dontRequireSortedTransactionsAfter + + val estimators = MiningConstraints(minerSettings, blockchainUpdater, height) + val mdConstraint = MultiDimensionalMiningConstraint(estimators.total, estimators.keyBlock) + val (unconfirmed, updatedMdConstraint) = utx.packUnconfirmed(mdConstraint, sortInBlock) + + val features = + if (version <= 2) Set.empty[Short] + else + settings.featuresSettings.supported + .filterNot(blockchainUpdater.approvedFeatures.keySet) + .filter(BlockchainFeatures.implemented) + .toSet + + log.debug(s"Adding ${unconfirmed.size} unconfirmed transaction(s) to new block") + Block.buildAndSign(version.toByte, currentTime, referencedBlockInfo.blockId, consensusData, unconfirmed, account, features) match { + case Left(e) => Left(e.err) + case Right(x) => Right((estimators, x, updatedMdConstraint.constraints.head)) + } } - } - } yield block - ) - } + } yield block + ) + }.delayExecution(delay) private def generateOneMicroBlockTask(account: PrivateKeyAccount, accumulatedBlock: Block, @@ -258,33 +242,16 @@ class MinerImpl(allChannels: ChannelGroup, } } - private def nextBlockGenerationTime(fs: FunctionalitySettings, - height: Int, - block: Block, - account: PublicKeyAccount): Either[String, (Long, Long)] = { - val balance = GeneratingBalanceProvider.balance(blockchainUpdater, fs, height, account.toAddress) - if (GeneratingBalanceProvider.isMiningAllowed(blockchainUpdater, height, balance)) { - val cData = block.consensusData - val blockGS = cData.generationSignature.arr - val baseTarget = cData.baseTarget - val calculatedTs = pos.validBlockDelay(blockGS, account.publicKey, baseTarget, balance) + block.timestamp - if (0 < calculatedTs && calculatedTs < Long.MaxValue) { - Right((balance, calculatedTs)) - } else - Left(s"Invalid next block generation time: $calculatedTs") - } else Left(s"Balance $balance of ${account.address} is lower than required for generation") - } - private def generateBlockTask(account: PrivateKeyAccount): Task[Unit] = { { - val height = blockchainUpdater.height - val blockForHit = (blockchainUpdater.blockAt(height - 100) orElse blockchainUpdater.lastBlock).get + val height = blockchainUpdater.height + val lastBlock = blockchainUpdater.lastBlock.get for { _ <- checkAge(height, blockchainUpdater.lastBlockTimestamp.get) _ <- Either.cond(blockchainUpdater.accountScript(account).isEmpty, (), s"Account(${account.toAddress}) is scripted and therefore not allowed to forge blocks") - balanceAndTs <- nextBlockGenerationTime(blockchainSettings.functionalitySettings, height, blockForHit, account) + balanceAndTs <- nextBlockGenerationTime(height, blockchainUpdater, blockchainSettings.functionalitySettings, lastBlock, account) (balance, ts) = balanceAndTs offset = calcOffset(timeService, ts, minerSettings.minimalBlockGenerationOffset) } yield (offset, balance) @@ -294,7 +261,7 @@ class MinerImpl(allChannels: ChannelGroup, nextBlockGenerationTimes += account.toAddress -> (System.currentTimeMillis() + offset.toMillis) generateOneBlockTask(account, balance)(offset).flatMap { case Right((estimators, block, totalConstraint)) => - BlockAppender(checkpoint, blockchainUpdater, timeService, utx, pos, settings, appenderScheduler)(block) + BlockAppender(checkpoint, blockchainUpdater, timeService, utx, settings, appenderScheduler)(block) .asyncBoundary(minerScheduler) .map { case Left(err) => log.warn("Error mining Block: " + err.toString) diff --git a/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index d1f0f62face..7ed8c8b4cc0 100644 --- a/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -1,7 +1,6 @@ package com.wavesplatform.state.appender import cats.data.EitherT -import com.wavesplatform.consensus.PoSCalculator import com.wavesplatform.metrics._ import com.wavesplatform.mining.Miner import com.wavesplatform.network._ @@ -26,7 +25,6 @@ object BlockAppender extends ScorexLogging with Instrumented { blockchainUpdater: BlockchainUpdater with Blockchain, time: Time, utxStorage: UtxPool, - pos: PoSCalculator, settings: WavesSettings, scheduler: Scheduler)(newBlock: Block): Task[Either[ValidationError, Option[BigInt]]] = Task { @@ -39,7 +37,7 @@ object BlockAppender extends ScorexLogging with Instrumented { (), BlockAppendError("Irrelevant block", newBlock)) _ = log.debug(s"Appending $newBlock") - maybeBaseHeight <- appendBlock(checkpoint, blockchainUpdater, utxStorage, pos, time, settings)(newBlock) + maybeBaseHeight <- appendBlock(checkpoint, blockchainUpdater, utxStorage, time, settings)(newBlock) } yield maybeBaseHeight map (_ => blockchainUpdater.score) } ) @@ -49,7 +47,6 @@ object BlockAppender extends ScorexLogging with Instrumented { blockchainUpdater: BlockchainUpdater with Blockchain, time: Time, utxStorage: UtxPool, - pos: PoSCalculator, settings: WavesSettings, allChannels: ChannelGroup, peerDatabase: PeerDatabase, @@ -59,7 +56,7 @@ object BlockAppender extends ScorexLogging with Instrumented { blockReceivingLag.safeRecord(System.currentTimeMillis() - newBlock.timestamp) (for { _ <- EitherT(Task.now(newBlock.signaturesValid())) - validApplication <- EitherT(apply(checkpoint, blockchainUpdater, time, utxStorage, pos, settings, scheduler)(newBlock)) + validApplication <- EitherT(apply(checkpoint, blockchainUpdater, time, utxStorage, settings, scheduler)(newBlock)) } yield validApplication).value.map { case Right(None) => // block already appended case Right(Some(_)) => diff --git a/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala b/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala index 52309b5f532..f41fdeede8d 100644 --- a/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala +++ b/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala @@ -1,6 +1,5 @@ package com.wavesplatform.state.appender -import com.wavesplatform.consensus.PoSCalculator import com.wavesplatform.metrics.{BlockStats, Instrumented, Metrics} import com.wavesplatform.mining.Miner import com.wavesplatform.network.{InvalidBlockStorage, PeerDatabase, formatBlocks, id} @@ -24,7 +23,6 @@ object ExtensionAppender extends ScorexLogging with Instrumented { def apply(checkpoint: CheckpointService, blockchainUpdater: BlockchainUpdater with Blockchain, utxStorage: UtxPool, - pos: PoSCalculator, time: Time, settings: WavesSettings, invalidBlocks: InvalidBlockStorage, @@ -45,10 +43,9 @@ object ExtensionAppender extends ScorexLogging with Instrumented { val forkApplicationResultEi = Coeval { extension.view .map { b => - b -> appendBlock(checkpoint, blockchainUpdater, utxStorage, pos, time, settings)(b).right - .map { - _.foreach(bh => BlockStats.applied(b, BlockStats.Source.Ext, bh)) - } + b -> appendBlock(checkpoint, blockchainUpdater, utxStorage, time, settings)(b).right.map { + _.foreach(bh => BlockStats.applied(b, BlockStats.Source.Ext, bh)) + } } .zipWithIndex .collectFirst { case ((b, Left(e)), i) => (i, b, e) } diff --git a/src/main/scala/com/wavesplatform/state/appender/package.scala b/src/main/scala/com/wavesplatform/state/appender/package.scala index 0c667a2cd22..ab5cea3cb91 100644 --- a/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -1,15 +1,17 @@ package com.wavesplatform.state -import com.wavesplatform.consensus.{GeneratingBalanceProvider, PoSCalculator} +import com.wavesplatform.features.BlockchainFeatures +import com.wavesplatform.features.FeatureProvider._ import com.wavesplatform.mining._ import com.wavesplatform.network._ -import com.wavesplatform.settings.WavesSettings +import com.wavesplatform.settings.{FunctionalitySettings, WavesSettings} import com.wavesplatform.utx.UtxPool import io.netty.channel.Channel import io.netty.channel.group.ChannelGroup import monix.eval.Task import scorex.block.Block import scorex.consensus.TransactionsOrdering +import scorex.transaction.PoSCalc._ import scorex.transaction.ValidationError.{BlockAppendError, BlockFromFuture, GenericError} import scorex.transaction._ import scorex.utils.{ScorexLogging, Time} @@ -47,10 +49,20 @@ package object appender extends ScorexLogging { } } + private def validateEffectiveBalance(blockchain: Blockchain, fs: FunctionalitySettings, block: Block, baseHeight: Int)( + effectiveBalance: Long): Either[String, Long] = + Either.cond( + block.timestamp < fs.minimalGeneratingBalanceAfter || + (block.timestamp >= fs.minimalGeneratingBalanceAfter && effectiveBalance >= MinimalEffectiveBalanceForGenerator1) || + blockchain.featureActivationHeight(BlockchainFeatures.SmallerMinimalGeneratingBalance.id).exists(baseHeight >= _) + && effectiveBalance >= MinimalEffectiveBalanceForGenerator2, + effectiveBalance, + s"generator's effective balance $effectiveBalance is less that required for generation" + ) + private[appender] def appendBlock(checkpoint: CheckpointService, blockchainUpdater: BlockchainUpdater with Blockchain, utxStorage: UtxPool, - pos: PoSCalculator, time: Time, settings: WavesSettings)(block: Block): Either[ValidationError, Option[Int]] = for { @@ -64,17 +76,9 @@ package object appender extends ScorexLogging { (), BlockAppendError(s"Account(${block.sender.toAddress}) is scripted are therefore not allowed to forge blocks", block) ) - _ <- blockConsensusValidation(blockchainUpdater, settings, pos, time.correctedTime(), block) { height => - val balance = GeneratingBalanceProvider.balance(blockchainUpdater, settings.blockchainSettings.functionalitySettings, height, block.sender) - Either.cond( - GeneratingBalanceProvider.isEffectiveBalanceValid(blockchainUpdater, - settings.blockchainSettings.functionalitySettings, - height, - block, - balance), - balance, - s"generator's effective balance $balance is less that required for generation" - ) + _ <- blockConsensusValidation(blockchainUpdater, settings, time.correctedTime(), block) { height => + val balance = PoSCalc.generatingBalance(blockchainUpdater, settings.blockchainSettings.functionalitySettings, block.sender, height) + validateEffectiveBalance(blockchainUpdater, settings.blockchainSettings.functionalitySettings, block, height)(balance) } baseHeight = blockchainUpdater.height maybeDiscardedTxs <- blockchainUpdater.processBlock(block) @@ -86,7 +90,7 @@ package object appender extends ScorexLogging { maybeDiscardedTxs.map(_ => baseHeight) } - private def blockConsensusValidation(blockchain: Blockchain, settings: WavesSettings, pos: PoSCalculator, currentTs: Long, block: Block)( + private def blockConsensusValidation(blockchain: Blockchain, settings: WavesSettings, currentTs: Long, block: Block)( genBalance: Int => Either[String, Long]): Either[ValidationError, Unit] = { val bcs = settings.blockchainSettings @@ -115,7 +119,7 @@ package object appender extends ScorexLogging { prevBlockData = parent.consensusData blockData = block.consensusData ggp = blockchain.parent(parent, 2) - cbt = pos.baseTarget(bcs.genesisSettings.averageBlockDelay.toSeconds, + cbt = calcBaseTarget(bcs.genesisSettings.averageBlockDelay, height, parent.consensusData.baseTarget, parent.timestamp, @@ -123,7 +127,7 @@ package object appender extends ScorexLogging { blockTime) bbt = blockData.baseTarget _ <- Either.cond(cbt == bbt, (), GenericError(s"declared baseTarget $bbt does not match calculated baseTarget $cbt")) - calcGs = pos.generatorSignature(prevBlockData.generationSignature.arr, generator.publicKey) + calcGs = calcGeneratorSignature(prevBlockData, generator) blockGs = blockData.generationSignature.arr _ <- Either.cond( calcGs.sameElements(blockGs), @@ -132,12 +136,12 @@ package object appender extends ScorexLogging { s"declared generation signature ${blockData.generationSignature.base58} does not match calculated generation signature ${ByteStr(calcGs).base58}") ) effectiveBalance <- genBalance(height).left.map(GenericError(_)) - minValidBlockTime = parent.timestamp + pos.validBlockDelay(blockGs, parent.consensusData.baseTarget, effectiveBalance) + hit = calcHit(prevBlockData, generator) + target = calcTarget(parent.timestamp, parent.consensusData.baseTarget, blockTime, effectiveBalance) _ <- Either.cond( - blockTime > minValidBlockTime - || (height == height1 && block.uniqueId == correctBlockId1) || (height == height2 && block.uniqueId == correctBlockId2), + hit < target || (height == height1 && block.uniqueId == correctBlockId1) || (height == height2 && block.uniqueId == correctBlockId2), (), - GenericError(s"calculated time $minValidBlockTime < block time $blockTime") + GenericError(s"calculated hit $hit >= calculated target $target") ) } yield () diff --git a/src/main/scala/scorex/api/http/AddressApiRoute.scala b/src/main/scala/scorex/api/http/AddressApiRoute.scala index 91f1e03ae03..e15b181232c 100644 --- a/src/main/scala/scorex/api/http/AddressApiRoute.scala +++ b/src/main/scala/scorex/api/http/AddressApiRoute.scala @@ -4,7 +4,6 @@ import java.nio.charset.StandardCharsets import akka.http.scaladsl.marshalling.ToResponseMarshallable import akka.http.scaladsl.server.Route -import com.wavesplatform.consensus.GeneratingBalanceProvider import com.wavesplatform.crypto import com.wavesplatform.settings.{FunctionalitySettings, RestAPISettings} import com.wavesplatform.state.Blockchain @@ -17,9 +16,9 @@ import play.api.libs.json._ import scorex.BroadcastRoute import scorex.account.{Address, PublicKeyAccount} import scorex.crypto.encode.Base58 -import scorex.transaction.{TransactionFactory, ValidationError} import scorex.transaction.ValidationError.GenericError import scorex.transaction.smart.script.ScriptCompiler +import scorex.transaction.{PoSCalc, TransactionFactory, ValidationError} import scorex.utils.Time import scorex.wallet.Wallet @@ -359,7 +358,7 @@ case class AddressApiRoute(settings: RestAPISettings, BalanceDetails( account.address, portfolio.balance, - GeneratingBalanceProvider.balance(blockchain, functionalitySettings, blockchain.height, account), + PoSCalc.generatingBalance(blockchain, functionalitySettings, account, blockchain.height), portfolio.balance - portfolio.lease.out, portfolio.effectiveBalance ) diff --git a/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala b/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala index b525a637b18..39fd1e5185e 100755 --- a/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala +++ b/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala @@ -1,7 +1,6 @@ package scorex.consensus.nxt.api.http import akka.http.scaladsl.server.Route -import com.wavesplatform.consensus.GeneratingBalanceProvider import com.wavesplatform.settings.{FunctionalitySettings, RestAPISettings} import com.wavesplatform.state.Blockchain import io.swagger.annotations._ @@ -9,6 +8,7 @@ import javax.ws.rs.Path import play.api.libs.json.Json import scorex.account.Address import scorex.api.http.{ApiRoute, CommonApiFunctions, InvalidAddress} +import scorex.transaction.PoSCalc @Path("/consensus") @Api(value = "/consensus") @@ -31,7 +31,8 @@ case class NxtConsensusApiRoute(settings: RestAPISettings, blockchain: Blockchai Address.fromString(address) match { case Left(_) => complete(InvalidAddress) case Right(account) => - complete(Json.obj("address" -> account.address, "balance" -> GeneratingBalanceProvider.balance(blockchain, fs, blockchain.height, account))) + val b = blockchain + complete(Json.obj("address" -> account.address, "balance" -> PoSCalc.generatingBalance(b, fs, account, b.height))) } } diff --git a/src/main/scala/scorex/transaction/PoSCalc.scala b/src/main/scala/scorex/transaction/PoSCalc.scala new file mode 100644 index 00000000000..0912fc19d8e --- /dev/null +++ b/src/main/scala/scorex/transaction/PoSCalc.scala @@ -0,0 +1,109 @@ +package scorex.transaction + +import com.wavesplatform.crypto +import com.wavesplatform.features.BlockchainFeatures +import com.wavesplatform.features.FeatureProvider._ +import com.wavesplatform.settings.FunctionalitySettings +import com.wavesplatform.state.Blockchain +import scorex.account.{Address, PublicKeyAccount} +import scorex.block.Block +import scorex.consensus.nxt.NxtLikeConsensusBlockData +import scorex.utils.ScorexLogging + +import scala.concurrent.duration.FiniteDuration + +object PoSCalc extends ScorexLogging { + + val MinimalEffectiveBalanceForGenerator1: Long = 1000000000000L + val MinimalEffectiveBalanceForGenerator2: Long = 100000000000L + + private val AvgBlockTimeDepth: Int = 3 + + // Min BaseTarget value is 9 because only in this case it is possible to get to next integer value (10) + // then increasing base target by 11% and casting it to Long afterward (see lines 55 and 59) + private val MinBaseTarget: Long = 9 + + private val MinBlockDelaySeconds = 53 + private val MaxBlockDelaySeconds = 67 + private val BaseTargetGamma = 64 + + def calcTarget(prevBlockTimestamp: Long, prevBlockBaseTarget: Long, timestamp: Long, balance: Long): BigInt = { + val eta = (timestamp - prevBlockTimestamp) / 1000 + BigInt(prevBlockBaseTarget) * eta * balance + } + + def calcHit(lastBlockData: NxtLikeConsensusBlockData, generator: PublicKeyAccount): BigInt = + BigInt(1, calcGeneratorSignature(lastBlockData, generator).take(8).reverse) + + def calcGeneratorSignature(lastBlockData: NxtLikeConsensusBlockData, generator: PublicKeyAccount): Array[Byte] = + crypto.fastHash(lastBlockData.generationSignature.arr ++ generator.publicKey) + + def calcBaseTarget(avgBlockDelay: FiniteDuration, + parentHeight: Int, + parentBaseTarget: Long, + parentTimestamp: Long, + maybeGreatGrandParentTimestamp: Option[Long], + timestamp: Long): Long = { + val avgDelayInSeconds = avgBlockDelay.toSeconds + + val prevBaseTarget = parentBaseTarget + if (parentHeight % 2 == 0) { + val blocktimeAverage = maybeGreatGrandParentTimestamp.fold(timestamp - parentTimestamp)(ggpts => (timestamp - ggpts) / AvgBlockTimeDepth) / 1000 + val minBlocktimeLimit = normalize(MinBlockDelaySeconds, avgDelayInSeconds) + val maxBlocktimeLimit = normalize(MaxBlockDelaySeconds, avgDelayInSeconds) + val baseTargetGamma = normalize(BaseTargetGamma, avgDelayInSeconds) + + val baseTarget = (if (blocktimeAverage > avgDelayInSeconds) { + prevBaseTarget * Math.min(blocktimeAverage, maxBlocktimeLimit) / avgDelayInSeconds + } else { + prevBaseTarget - prevBaseTarget * baseTargetGamma * + (avgDelayInSeconds - Math.max(blocktimeAverage, minBlocktimeLimit)) / (avgDelayInSeconds * 100) + }).toLong + + normalizeBaseTarget(baseTarget, avgDelayInSeconds) + } else { + prevBaseTarget + } + } + + def generatingBalance(blockchain: Blockchain, fs: FunctionalitySettings, account: Address, atHeight: Int): Long = { + val generatingBalanceDepth = fs.generatingBalanceDepth(atHeight) + blockchain.effectiveBalance(account, atHeight, generatingBalanceDepth) + } + + def nextBlockGenerationTime(height: Int, + blockchain: Blockchain, + fs: FunctionalitySettings, + block: Block, + account: PublicKeyAccount): Either[String, (Long, Long)] = { + val balance = generatingBalance(blockchain, fs, account, height) + Either + .cond( + (!blockchain + .isFeatureActivated(BlockchainFeatures.SmallerMinimalGeneratingBalance, height) && balance >= MinimalEffectiveBalanceForGenerator1) || + (blockchain + .isFeatureActivated(BlockchainFeatures.SmallerMinimalGeneratingBalance, height) && balance >= MinimalEffectiveBalanceForGenerator2), + balance, + s"Balance $balance of ${account.address} is lower than required for generation" + ) + .flatMap { _ => + val cData = block.consensusData + val hit = calcHit(cData, account) + val t = cData.baseTarget + val calculatedTs = (hit * 1000) / (BigInt(t) * balance) + block.timestamp + if (0 < calculatedTs && calculatedTs < Long.MaxValue) { + Right((balance, calculatedTs.toLong)) + } else { + Left(s"Invalid next block generation time: $calculatedTs") + } + } + } + + private def normalizeBaseTarget(bt: Long, averageBlockDelaySeconds: Long): Long = { + val maxBaseTarget = Long.MaxValue / averageBlockDelaySeconds + if (bt < MinBaseTarget) MinBaseTarget else if (bt > maxBaseTarget) maxBaseTarget else bt + } + + private def normalize(value: Long, averageBlockDelaySeconds: Long): Double = value * averageBlockDelaySeconds / (60: Double) + +} diff --git a/src/test/resources/genesis.dev.conf b/src/test/resources/genesis.dev.conf deleted file mode 100644 index 2f7af5b1f56..00000000000 --- a/src/test/resources/genesis.dev.conf +++ /dev/null @@ -1,21 +0,0 @@ -# Configuration for genesis block generator -# To generate run from SBT: test:run src/test/resources/genesis.it.conf -genesis-generator { - network-type: "D" - - initial-balance: 10000000000000000 - base-target: 100 - average-block-delay: 60s - timestamp: 1489352400000 # Comment to use the current time - - # seed text -> share - # the sum of shares should be <= initial-balance - distributions { - "create genesis wallet devnet-0": 200000000000000 - "create genesis wallet devnet-0-d": 500000000000000 - "create genesis wallet devnet-1": 1000000000000000 - "create genesis wallet devnet-1-d": 1500000000000000 - "create genesis wallet devnet-2": 2000000000000000 - "create genesis wallet devnet-2-d": 2500000000000000 - } -} diff --git a/src/test/resources/genesis.it.conf b/src/test/resources/genesis.it.conf index b0b2368dd2c..0cd33f6b61f 100644 --- a/src/test/resources/genesis.it.conf +++ b/src/test/resources/genesis.it.conf @@ -22,4 +22,4 @@ genesis-generator { "node09": 60000000000000 "node10": 830500000000000 } -} \ No newline at end of file +} diff --git a/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala b/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala deleted file mode 100644 index 35c63e8e680..00000000000 --- a/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.wavesplatform.consensus - -import org.scalatest.{Matchers, PropSpec} -import scala.util.Random - -class FairPoSCalculatorTest extends PropSpec with Matchers { - - val pos = FairPoSCalculator - - case class Block(height: Int, baseTarget: Long, timestamp: Long, delay: Long) - - def generationSignature: Array[Byte] = { - val arr = new Array[Byte](32) - Random.nextBytes(arr) - arr - } - - property("Correct consensus parameters of blocks generated with FairPoS") { - - val balance = 50000000L * 100000000L - - val blockDelaySeconds = 60 - - val defaultBaseTarget = 100L - - val first = Block(0, defaultBaseTarget, System.currentTimeMillis(), 0) - - val chain = (1 to 10000 foldLeft List(first))((acc, _) => { - acc match { - case last :: _ => - val delay = pos.validBlockDelay(generationSignature, last.baseTarget, balance) - val bt = pos.baseTarget( - blockDelaySeconds, - last.height + 1, - last.baseTarget, - last.timestamp, - if (acc.isDefinedAt(2)) Some(acc(2).timestamp) else None, - last.timestamp + delay - ) - - Block( - last.height + 1, - bt, - last.timestamp + delay, - delay - ) :: acc - - case _ => ??? - } - - }).reverse - - val maxBT = chain.maxBy(_.baseTarget).baseTarget - val avgBT = chain.map(_.baseTarget).sum / chain.length - val minBT = chain.minBy(_.baseTarget).baseTarget - - val maxDelay = chain.tail.maxBy(_.delay).delay - val avgDelay = chain.tail.map(_.delay).sum / (chain.length - 1) - val minDelay = chain.tail.minBy(_.delay).delay - - print( - s""" - |BT: $minBT $avgBT $maxBT - |Delay: $minDelay $avgDelay $maxDelay - """.stripMargin - ) - - assert(avgDelay < 80000 && avgDelay > 40000) - assert(avgBT < 200 && avgBT > 20) - } - -} diff --git a/waves-devnet.conf b/waves-devnet.conf index 39aa3eb5cec..2af84dceab1 100644 --- a/waves-devnet.conf +++ b/waves-devnet.conf @@ -27,17 +27,17 @@ waves { genesis { average-block-delay: 60000ms initial-base-target: 100 - timestamp: 1489352400000 - block-timestamp: 1489352400000 - signature: "4fKa1vwqnuAoUEX2LJGaKKYZzQC3KYT3PDwash33RDP21GawXkFPJJKG6joScodQE5UXa5mymB6bomYFHpdS4YT4" + timestamp: 1500635421931 + block-timestamp: 1500635421931 + signature: "GxifkzjW43Cg3xjpNjk5EwhVh5q9EN82WQpGMfNY33c1eCmLFtZGHARwRZLaSZaYss7iTt9yavTBWcXqagCBXii" initial-balance: 10000000000000000 transactions = [ - {recipient: "3FR3MyuXumwBj1bLC8xnW38iHtwm9Ugdm8K", amount: 2500000000000000}, - {recipient: "3FgScYB6MNdnN8m4xXddQe1Bjkwmd3U7YtM", amount: 200000000000000}, - {recipient: "3FWXhvWq2r8m54MmCEZ3YZkLg2qUdGWbU3V", amount: 1000000000000000}, - {recipient: "3FkBWsgT9T3snZ4ZpzzQCJWQngJBLdDEPfU", amount: 500000000000000}, + {recipient: "3FR3MyuXumwBj1bLC8xnW38iHtwm9Ugdm8K", amount: 1500000000000000}, + {recipient: "3FgScYB6MNdnN8m4xXddQe1Bjkwmd3U7YtM", amount: 1500000000000000}, + {recipient: "3FWXhvWq2r8m54MmCEZ3YZkLg2qUdGWbU3V", amount: 1500000000000000}, + {recipient: "3FkBWsgT9T3snZ4ZpzzQCJWQngJBLdDEPfU", amount: 1500000000000000}, {recipient: "3FeeqPbaEUQ8h3eQ4ZX9WcqzqskGbfTqM2a", amount: 1500000000000000}, - {recipient: "3FcSgww3tKZ7feQVmcnPFmRxsjqBodYz63x", amount: 2000000000000000} + {recipient: "3FcSgww3tKZ7feQVmcnPFmRxsjqBodYz63x", amount: 1500000000000000} ] } } From 0760ccd5ac4e9b9a402c6fbea8d269d67cf0bd42 Mon Sep 17 00:00:00 2001 From: Alexandr Date: Fri, 18 May 2018 02:21:17 +0300 Subject: [PATCH 04/52] Revert "Revert "Node 129 fair proof of stake"" This reverts commit 466da3cd378f51039bc158d98485a8173c6ece10. --- .../wavesplatform/it/BaseTargetChecker.scala | 19 ++- .../wavesplatform/it/api/SyncHttpApi.scala | 6 + .../it/sync/FairPoSTestSuite.scala | 49 ++++++ .../scala/com/wavesplatform/Application.scala | 11 +- .../scala/com/wavesplatform/Importer.scala | 4 +- .../consensus/GeneratingBalanceProvider.scala | 30 ++++ .../consensus/PoSCalculator.scala | 138 ++++++++++++++++ .../features/BlockchainFeature.scala | 4 +- .../com/wavesplatform/mining/Miner.scala | 151 +++++++++++------- .../state/appender/BlockAppender.scala | 7 +- .../state/appender/ExtensionAppender.scala | 9 +- .../state/appender/package.scala | 46 +++--- .../scorex/api/http/AddressApiRoute.scala | 5 +- .../nxt/api/http/NxtConsensusApiRoute.scala | 5 +- .../scala/scorex/transaction/PoSCalc.scala | 109 ------------- src/test/resources/genesis.dev.conf | 21 +++ src/test/resources/genesis.it.conf | 2 +- .../consensus/FairPoSCalculatorTest.scala | 72 +++++++++ waves-devnet.conf | 16 +- 19 files changed, 478 insertions(+), 226 deletions(-) create mode 100644 it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala create mode 100644 src/main/scala/com/wavesplatform/consensus/GeneratingBalanceProvider.scala create mode 100644 src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala delete mode 100644 src/main/scala/scorex/transaction/PoSCalc.scala create mode 100644 src/test/resources/genesis.dev.conf create mode 100644 src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala diff --git a/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala b/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala index 39b64e27c05..77b51a5904c 100644 --- a/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala +++ b/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala @@ -3,6 +3,7 @@ package com.wavesplatform.it import java.time.Instant import com.typesafe.config.ConfigFactory.{defaultApplication, defaultReference} +import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.db.openDB import com.wavesplatform.history.StorageFactory import com.wavesplatform.settings._ @@ -10,36 +11,34 @@ import com.wavesplatform.state.{ByteStr, EitherExt2} import net.ceedubs.ficus.Ficus._ import scorex.account.PublicKeyAccount import scorex.block.Block -import scorex.transaction.PoSCalc import scorex.utils.NTP object BaseTargetChecker { def main(args: Array[String]): Unit = { - val startTs = System.currentTimeMillis() - val docker = Docker(getClass) + val docker = Docker(getClass) val sharedConfig = docker.genesisOverride .withFallback(docker.configTemplate) .withFallback(defaultApplication()) .withFallback(defaultReference()) .resolve() val settings = WavesSettings.fromConfig(sharedConfig) - val fs = settings.blockchainSettings.functionalitySettings val genesisBlock = Block.genesis(settings.blockchainSettings.genesisSettings).explicitGet() val db = openDB("/tmp/tmp-db", 1024) val bu = StorageFactory(settings, db, NTP) + val pos = new PoSSelector(bu) bu.processBlock(genesisBlock) println(s"Genesis TS = ${Instant.ofEpochMilli(genesisBlock.timestamp)}") val m = NodeConfigs.Default.map(_.withFallback(sharedConfig)).collect { case cfg if cfg.as[Boolean]("waves.miner.enable") => - val publicKey = PublicKeyAccount(cfg.as[ByteStr]("public-key").arr) - val address = publicKey.toAddress - PoSCalc.nextBlockGenerationTime(1, bu, fs, genesisBlock, publicKey) match { - case Right((_, ts)) => f"$address: ${(ts - startTs) * 1e-3}%10.3f s" - case _ => s"$address: n/a" - } + val account = PublicKeyAccount(cfg.as[ByteStr]("public-key").arr) + val address = account.toAddress + val balance = bu.balance(address, None) + val consensus = genesisBlock.consensusData + val timeDelay = pos.validBlockDelay(consensus.generationSignature.arr, account.publicKey, consensus.baseTarget, balance) + f"$address: ${timeDelay * 1e-3}%10.3f s" } docker.close() diff --git a/it/src/main/scala/com/wavesplatform/it/api/SyncHttpApi.scala b/it/src/main/scala/com/wavesplatform/it/api/SyncHttpApi.scala index 01c068912e3..0fac216a7e3 100644 --- a/it/src/main/scala/com/wavesplatform/it/api/SyncHttpApi.scala +++ b/it/src/main/scala/com/wavesplatform/it/api/SyncHttpApi.scala @@ -61,6 +61,9 @@ object SyncHttpApi extends Assertions { def postJsonWithApiKey[A: Writes](path: String, body: A): Response = Await.result(async(n).postJsonWithApiKey(path, body), RequestAwaitTime) + def accountBalance(acc: String): Long = + Await.result(async(n).accountBalance(acc), RequestAwaitTime) + def accountBalances(acc: String): (Long, Long) = Await.result(async(n).accountBalances(acc), RequestAwaitTime) @@ -164,6 +167,9 @@ object SyncHttpApi extends Assertions { def height: Int = Await.result(async(n).height, RequestAwaitTime) + + def blockHeadersSeq(from: Int, to: Int): Seq[BlockHeaders] = + Await.result(async(n).blockHeadersSeq(from, to), RequestAwaitTime) } implicit class NodesExtSync(nodes: Seq[Node]) { diff --git a/it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala new file mode 100644 index 00000000000..c8dc7e88c4a --- /dev/null +++ b/it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala @@ -0,0 +1,49 @@ +package com.wavesplatform.it.sync + +import com.typesafe.config.{Config, ConfigFactory} +import org.scalatest.{CancelAfterFailure, FunSuite} +import com.wavesplatform.it.api.State +import com.wavesplatform.it.api.SyncHttpApi._ +import com.wavesplatform.it.transactions.NodesFromDocker +import com.wavesplatform.it.util._ +import scala.concurrent.duration._ + +class FairPoSTestSuite extends FunSuite with CancelAfterFailure with NodesFromDocker { + import FairPoSTestSuite._ + + override protected def nodeConfigs: Seq[Config] = Configs + + private val transferFee = 0.001.waves + private val transferAmount = 1000.waves + + test("blockchain grows with FairPoS activated") { + nodes.head.waitForHeight(10, 3.minutes) + + val txId = nodes.head.transfer(nodes.head.address, nodes.last.address, transferAmount, transferFee).id + nodes.last.waitForTransaction(txId) + + val heightAfterTransfer = nodes.head.height + + nodes.head.waitForHeight(heightAfterTransfer + 20, 5.minutes) + } +} + +object FairPoSTestSuite { + import com.wavesplatform.it.NodeConfigs._ + private val microblockActivationHeight = 0 + private val fairPoSActivationHeight = 10 + + private val config = + ConfigFactory.parseString(s""" + |waves { + | blockchain.custom { + | functionality { + | pre-activated-features {1 = $microblockActivationHeight, 8 = $fairPoSActivationHeight} + | generation-balance-depth-from-50-to-1000-after-height = 1000 + | } + | } + | miner.quorum = 1 + |}""".stripMargin) + + val Configs: Seq[Config] = Default.map(config.withFallback(_)).take(4) +} diff --git a/src/main/scala/com/wavesplatform/Application.scala b/src/main/scala/com/wavesplatform/Application.scala index c462d84da49..0437750a8de 100644 --- a/src/main/scala/com/wavesplatform/Application.scala +++ b/src/main/scala/com/wavesplatform/Application.scala @@ -11,6 +11,7 @@ import akka.stream.ActorMaterializer import cats.instances.all._ import com.typesafe.config._ import com.wavesplatform.actor.RootActorSystem +import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.db.openDB import com.wavesplatform.features.api.ActivationApiRoute import com.wavesplatform.history.{CheckpointServiceImpl, StorageFactory} @@ -123,19 +124,25 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con maybeUtx = Some(utxStorage) val knownInvalidBlocks = new InvalidBlockStorageImpl(settings.synchronizationSettings.invalidBlocksStorage) + + val pos = new PoSSelector(blockchainUpdater) + val miner = if (settings.minerSettings.enable) - new MinerImpl(allChannels, blockchainUpdater, checkpointService, settings, time, utxStorage, wallet, minerScheduler, appenderScheduler) + new MinerImpl(allChannels, blockchainUpdater, checkpointService, settings, time, utxStorage, wallet, pos, minerScheduler, appenderScheduler) else Miner.Disabled val processBlock = - BlockAppender(checkpointService, blockchainUpdater, time, utxStorage, settings, allChannels, peerDatabase, miner, appenderScheduler) _ + BlockAppender(checkpointService, blockchainUpdater, time, utxStorage, pos, settings, allChannels, peerDatabase, miner, appenderScheduler) _ + val processCheckpoint = CheckpointAppender(checkpointService, blockchainUpdater, blockchainUpdater, peerDatabase, miner, allChannels, appenderScheduler) _ + val processFork = ExtensionAppender( checkpointService, blockchainUpdater, utxStorage, + pos, time, settings, knownInvalidBlocks, diff --git a/src/main/scala/com/wavesplatform/Importer.scala b/src/main/scala/com/wavesplatform/Importer.scala index b9c8f4aa08a..f791c87c1fc 100644 --- a/src/main/scala/com/wavesplatform/Importer.scala +++ b/src/main/scala/com/wavesplatform/Importer.scala @@ -4,6 +4,7 @@ import java.io._ import com.google.common.primitives.Ints import com.typesafe.config.ConfigFactory +import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.db.openDB import com.wavesplatform.history.{CheckpointServiceImpl, StorageFactory} import com.wavesplatform.mining.MultiDimensionalMiningConstraint @@ -57,8 +58,9 @@ object Importer extends ScorexLogging { case Success(inputStream) => val db = openDB(settings.dataDirectory, settings.levelDbCacheSize) val blockchainUpdater = StorageFactory(settings, db, NTP) + val pos = new PoSSelector(blockchainUpdater) val checkpoint = new CheckpointServiceImpl(db, settings.checkpointsSettings) - val extAppender = BlockAppender(checkpoint, blockchainUpdater, NTP, utxPoolStub, settings, scheduler) _ + val extAppender = BlockAppender(checkpoint, blockchainUpdater, NTP, utxPoolStub, pos, settings, scheduler) _ checkGenesis(settings, blockchainUpdater) val bis = new BufferedInputStream(inputStream) var quit = false diff --git a/src/main/scala/com/wavesplatform/consensus/GeneratingBalanceProvider.scala b/src/main/scala/com/wavesplatform/consensus/GeneratingBalanceProvider.scala new file mode 100644 index 00000000000..39bf5aebc28 --- /dev/null +++ b/src/main/scala/com/wavesplatform/consensus/GeneratingBalanceProvider.scala @@ -0,0 +1,30 @@ +package com.wavesplatform.consensus + +import com.wavesplatform.features.BlockchainFeatures +import com.wavesplatform.settings.FunctionalitySettings +import com.wavesplatform.state.Blockchain +import scorex.account.Address +import scorex.block.Block + +object GeneratingBalanceProvider { + private val MinimalEffectiveBalanceForGenerator1: Long = 1000000000000L + private val MinimalEffectiveBalanceForGenerator2: Long = 100000000000L + private val FirstDepth = 50 + private val SecondDepth = 1000 + + def isMiningAllowed(blockchain: Blockchain, height: Int, effectiveBalance: Long): Boolean = { + val activated = blockchain.activatedFeatures.get(BlockchainFeatures.SmallerMinimalGeneratingBalance.id).exists(height >= _) + (!activated && effectiveBalance >= MinimalEffectiveBalanceForGenerator1) || (activated && effectiveBalance >= MinimalEffectiveBalanceForGenerator2) + } + + def isEffectiveBalanceValid(blockchain: Blockchain, fs: FunctionalitySettings, height: Int, block: Block, effectiveBalance: Long): Boolean = + block.timestamp < fs.minimalGeneratingBalanceAfter || (block.timestamp >= fs.minimalGeneratingBalanceAfter && effectiveBalance >= MinimalEffectiveBalanceForGenerator1) || + blockchain.activatedFeatures + .get(BlockchainFeatures.SmallerMinimalGeneratingBalance.id) + .exists(height >= _) && effectiveBalance >= MinimalEffectiveBalanceForGenerator2 + + def balance(blockchain: Blockchain, fs: FunctionalitySettings, height: Int, account: Address): Long = { + val depth = if (height >= fs.generationBalanceDepthFrom50To1000AfterHeight) SecondDepth else FirstDepth + blockchain.effectiveBalance(account, height, depth) + } +} diff --git a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala new file mode 100644 index 00000000000..5e438473d5e --- /dev/null +++ b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala @@ -0,0 +1,138 @@ +package com.wavesplatform.consensus + +import com.wavesplatform.crypto +import com.wavesplatform.features.BlockchainFeatures +import com.wavesplatform.features.FeatureProvider._ +import com.wavesplatform.state.Blockchain + +trait PoSCalculator { + protected val HitSize: Int = 8 + protected val MinBaseTarget: Long = 9 + + def generatorSignature(signature: Array[Byte], publicKey: Array[Byte]): Array[Byte] = { + val s = new Array[Byte](crypto.DigestSize * 2) + System.arraycopy(signature, 0, s, 0, crypto.DigestSize) + System.arraycopy(publicKey, 0, s, crypto.DigestSize, crypto.DigestSize) + crypto.fastHash(s) + } + + def baseTarget(targetBlockDelaySeconds: Long, + prevHeight: Int, + prevBaseTarget: Long, + parentTimestamp: Long, + maybeGreatGrandParentTimestamp: Option[Long], + timestamp: Long): Long + + protected def hit(generatorSignature: Array[Byte]): BigInt = BigInt(1, generatorSignature.take(HitSize).reverse) + + def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long + + def validBlockDelay(genSig: Array[Byte], baseTarget: Long, balance: Long): Long = { + calculateDelay(hit(genSig), baseTarget, balance) + } + + def validBlockDelay(genSig: Array[Byte], publicKey: Array[Byte], baseTarget: Long, balance: Long): Long = { + calculateDelay(hit(generatorSignature(genSig, publicKey)), baseTarget, balance) + } + + protected def normalize(value: Long, targetBlockDelaySeconds: Long): Double = + value * targetBlockDelaySeconds / (60: Double) + + protected def normalizeBaseTarget(baseTarget: Long, targetBlockDelaySeconds: Long): Long = { + baseTarget + .max(MinBaseTarget) + .min(Long.MaxValue / targetBlockDelaySeconds) + } + +} + +class PoSSelector(val blockchain: Blockchain) extends PoSCalculator { + + protected def pos: PoSCalculator = + if (fair(blockchain.height)) FairPoSCalculator + else NxtPoSCalculator + + override def baseTarget(targetBlockDelaySeconds: Long, + prevHeight: Int, + prevBaseTarget: Long, + parentTimestamp: Long, + maybeGreatGrandParentTimestamp: Option[Long], + timestamp: Long): Long = { + pos.baseTarget(targetBlockDelaySeconds, prevHeight, prevBaseTarget, parentTimestamp, maybeGreatGrandParentTimestamp, timestamp) + } + + def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long = + pos.calculateDelay(hit, bt, balance) + + private def fair(height: Int): Boolean = blockchain.activatedFeaturesAt(height).contains(BlockchainFeatures.FairPoS.id) +} + +object NxtPoSCalculator extends PoSCalculator { + protected val MinBlockDelaySeconds = 53 + protected val MaxBlockDelaySeconds = 67 + protected val BaseTargetGamma = 64 + protected val MeanCalculationDepth = 3 + + def baseTarget(targetBlockDelaySeconds: Long, + prevHeight: Int, + prevBaseTarget: Long, + parentTimestamp: Long, + maybeGreatGrandParentTimestamp: Option[Long], + timestamp: Long): Long = { + + if (prevHeight % 2 == 0) { + val meanBlockDelay = maybeGreatGrandParentTimestamp.fold(timestamp - parentTimestamp)(ts => (timestamp - ts) / MeanCalculationDepth) / 1000 + val minBlockDelay = normalize(MinBlockDelaySeconds, targetBlockDelaySeconds) + val maxBlockDelay = normalize(MaxBlockDelaySeconds, targetBlockDelaySeconds) + val baseTargetGamma = normalize(BaseTargetGamma, targetBlockDelaySeconds) + + val baseTarget = (if (meanBlockDelay > targetBlockDelaySeconds) { + prevBaseTarget * Math.min(meanBlockDelay, maxBlockDelay) / targetBlockDelaySeconds + } else { + prevBaseTarget - prevBaseTarget * baseTargetGamma * + (targetBlockDelaySeconds - Math.max(meanBlockDelay, minBlockDelay)) / (targetBlockDelaySeconds * 100) + }).toLong + + normalizeBaseTarget(baseTarget, targetBlockDelaySeconds) + } else { + prevBaseTarget + } + } + + def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long = ((hit * 1000) / (BigInt(bt) * balance)).toLong + +} + +object FairPoSCalculator extends PoSCalculator { + private val MaxSignature: Array[Byte] = Array.fill[Byte](HitSize)(-1) + private val MaxHit: BigDecimal = BigDecimal(BigInt(1, MaxSignature)) + private val C1 = 70000 + private val C2 = 5E17 + private val TMin = 5000 + + def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long = { + val h = (BigDecimal(hit) / MaxHit).toDouble + val a = TMin + C1 * math.log(1 - C2 * math.log(h) / bt / balance) + a.toLong + } + + def baseTarget(targetBlockDelaySeconds: Long, + prevHeight: Int, + prevBaseTarget: Long, + parentTimestamp: Long, + maybeGreatGrandParentTimestamp: Option[Long], + timestamp: Long): Long = { + val maxDelay = normalize(90, targetBlockDelaySeconds) + val minDelay = normalize(30, targetBlockDelaySeconds) + + maybeGreatGrandParentTimestamp match { + case None => + prevBaseTarget + case Some(ts) => + val avg = (timestamp - ts) / 3 / 1000 + if (avg > maxDelay) prevBaseTarget + math.max(1, prevBaseTarget / 100) + else if (avg < minDelay) prevBaseTarget - math.max(1, prevBaseTarget / 100) + else prevBaseTarget + } + } +} diff --git a/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala b/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala index 8c9d3e31f8a..c2215b53d4a 100644 --- a/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala +++ b/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala @@ -11,6 +11,7 @@ object BlockchainFeatures { val DataTransaction = BlockchainFeature(5, "Data Transaction") val BurnAnyTokens = BlockchainFeature(6, "Burn Any Tokens") val FeeSponsorship = BlockchainFeature(7, "Fee Sponsorship") + val FairPoS = BlockchainFeature(8, "Fair PoS") private val dict = Seq( SmallerMinimalGeneratingBalance, @@ -19,7 +20,8 @@ object BlockchainFeatures { SmartAccounts, DataTransaction, BurnAnyTokens, - FeeSponsorship + FeeSponsorship, + FairPoS ).map(f => f.id -> f).toMap val implemented: Set[Short] = dict.keySet diff --git a/src/main/scala/com/wavesplatform/mining/Miner.scala b/src/main/scala/com/wavesplatform/mining/Miner.scala index 09225a5fb6c..35cf37a137a 100644 --- a/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -1,11 +1,12 @@ package com.wavesplatform.mining import cats.data.EitherT +import com.wavesplatform.consensus.{GeneratingBalanceProvider, PoSCalculator} import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.features.FeatureProvider._ import com.wavesplatform.metrics.{BlockStats, HistogramExt, Instrumented} import com.wavesplatform.network._ -import com.wavesplatform.settings.WavesSettings +import com.wavesplatform.settings.{FunctionalitySettings, WavesSettings} import com.wavesplatform.state._ import com.wavesplatform.state.appender.{BlockAppender, MicroblockAppender} import com.wavesplatform.utx.UtxPool @@ -15,11 +16,10 @@ import kamon.metric.instrument import monix.eval.Task import monix.execution.cancelables.{CompositeCancelable, SerialCancelable} import monix.execution.schedulers.SchedulerService -import scorex.account.{Address, PrivateKeyAccount} +import scorex.account.{Address, PrivateKeyAccount, PublicKeyAccount} import scorex.block.Block._ import scorex.block.{Block, MicroBlock} import scorex.consensus.nxt.NxtLikeConsensusBlockData -import scorex.transaction.PoSCalc._ import scorex.transaction._ import scorex.utils.{ScorexLogging, Time} import scorex.wallet.Wallet @@ -59,6 +59,7 @@ class MinerImpl(allChannels: ChannelGroup, timeService: Time, utx: UtxPool, wallet: Wallet, + pos: PoSCalculator, val minerScheduler: SchedulerService, val appenderScheduler: SchedulerService) extends Miner @@ -100,60 +101,75 @@ class MinerImpl(allChannels: ChannelGroup, private def ngEnabled: Boolean = blockchainUpdater.featureActivationHeight(BlockchainFeatures.NG.id).exists(blockchainUpdater.height > _ + 1) private def generateOneBlockTask(account: PrivateKeyAccount, balance: Long)( - delay: FiniteDuration): Task[Either[String, (MiningConstraints, Block, MiningConstraint)]] = + delay: FiniteDuration): Task[Either[String, (MiningConstraints, Block, MiningConstraint)]] = { Task { - // should take last block right at the time of mining since microblocks might have been added - val height = blockchainUpdater.height - val version = if (height <= blockchainSettings.functionalitySettings.blockVersion3AfterHeight) PlainBlockVersion else NgBlockVersion - val lastBlock = blockchainUpdater.lastBlock.get - val greatGrandParentTimestamp = blockchainUpdater.parent(lastBlock, 2).map(_.timestamp) - val referencedBlockInfo = blockchainUpdater.bestLastBlockInfo(System.currentTimeMillis() - minMicroBlockDurationMills).get - val pc = allChannels.size() - lazy val currentTime = timeService.correctedTime() - lazy val h = calcHit(referencedBlockInfo.consensus, account) - lazy val t = calcTarget(referencedBlockInfo.timestamp, referencedBlockInfo.consensus.baseTarget, currentTime, balance) - measureSuccessful( - blockBuildTimeStats, - for { - _ <- Either.cond(pc >= minerSettings.quorum, - (), - s"Quorum not available ($pc/${minerSettings.quorum}, not forging block with ${account.address}") - _ <- Either.cond(h < t, (), s"${System.currentTimeMillis()}: Hit $h was NOT less than target $t, not forging block with ${account.address}") - _ = log.debug(s"Forging with ${account.address}, H $h < T $t, balance $balance, prev block ${referencedBlockInfo.blockId}") - _ = log.debug(s"Previous block ID ${referencedBlockInfo.blockId} at $height with target ${referencedBlockInfo.consensus.baseTarget}") - block <- { - val avgBlockDelay = blockchainSettings.genesisSettings.averageBlockDelay - val btg = calcBaseTarget(avgBlockDelay, - height, - referencedBlockInfo.consensus.baseTarget, - referencedBlockInfo.timestamp, - greatGrandParentTimestamp, - currentTime) - val gs = calcGeneratorSignature(referencedBlockInfo.consensus, account) - val consensusData = NxtLikeConsensusBlockData(btg, ByteStr(gs)) - val sortInBlock = blockchainUpdater.height <= blockchainSettings.functionalitySettings.dontRequireSortedTransactionsAfter - - val estimators = MiningConstraints(minerSettings, blockchainUpdater, height) - val mdConstraint = MultiDimensionalMiningConstraint(estimators.total, estimators.keyBlock) - val (unconfirmed, updatedMdConstraint) = utx.packUnconfirmed(mdConstraint, sortInBlock) - - val features = - if (version <= 2) Set.empty[Short] - else - settings.featuresSettings.supported - .filterNot(blockchainUpdater.approvedFeatures.keySet) - .filter(BlockchainFeatures.implemented) - .toSet - - log.debug(s"Adding ${unconfirmed.size} unconfirmed transaction(s) to new block") - Block.buildAndSign(version.toByte, currentTime, referencedBlockInfo.blockId, consensusData, unconfirmed, account, features) match { - case Left(e) => Left(e.err) - case Right(x) => Right((estimators, x, updatedMdConstraint.constraints.head)) - } - } - } yield block - ) + forgeBlock(account, balance) }.delayExecution(delay) + } + + private def forgeBlock(account: PrivateKeyAccount, balance: Long): Either[String, (MiningConstraints, Block, MiningConstraint)] = { + // should take last block right at the time of mining since microblocks might have been added + val height = blockchainUpdater.height + val version = if (height <= blockchainSettings.functionalitySettings.blockVersion3AfterHeight) PlainBlockVersion else NgBlockVersion + val lastBlock = blockchainUpdater.lastBlock.get + val greatGrandParentTimestamp = blockchainUpdater.parent(lastBlock, 2).map(_.timestamp) + val referencedBlockInfo = blockchainUpdater.bestLastBlockInfo(System.currentTimeMillis() - minMicroBlockDurationMills).get + val pc = allChannels.size() + lazy val currentTime = timeService.correctedTime() + lazy val blockDelay = currentTime - lastBlock.timestamp + lazy val validBlockDelay = pos.validBlockDelay( + referencedBlockInfo.consensus.generationSignature.arr, + account.publicKey, + referencedBlockInfo.consensus.baseTarget, + balance + ) + measureSuccessful( + blockBuildTimeStats, + for { + _ <- Either.cond(pc >= minerSettings.quorum, + (), + s"Quorum not available ($pc/${minerSettings.quorum}, not forging block with ${account.address}") + _ <- Either.cond( + blockDelay > validBlockDelay, + (), + s"${System.currentTimeMillis()}: Block delay $blockDelay was NOT less than estimated delay $validBlockDelay, not forging block with ${account.address}" + ) + _ = log.debug( + s"Forging with ${account.address}, Time $blockDelay > Estimated Time $validBlockDelay, balance $balance, prev block ${referencedBlockInfo.blockId}") + _ = log.debug(s"Previous block ID ${referencedBlockInfo.blockId} at $height with target ${referencedBlockInfo.consensus.baseTarget}") + block <- { + val avgBlockDelay = blockchainSettings.genesisSettings.averageBlockDelay + val btg = pos.baseTarget(avgBlockDelay.toSeconds, + height, + referencedBlockInfo.consensus.baseTarget, + referencedBlockInfo.timestamp, + greatGrandParentTimestamp, + currentTime) + val gs = pos.generatorSignature(referencedBlockInfo.consensus.generationSignature.arr, account.publicKey) + val consensusData = NxtLikeConsensusBlockData(btg, ByteStr(gs)) + val sortInBlock = blockchainUpdater.height <= blockchainSettings.functionalitySettings.dontRequireSortedTransactionsAfter + + val estimators = MiningConstraints(minerSettings, blockchainUpdater, height) + val mdConstraint = MultiDimensionalMiningConstraint(estimators.total, estimators.keyBlock) + val (unconfirmed, updatedMdConstraint) = utx.packUnconfirmed(mdConstraint, sortInBlock) + + val features = + if (version <= 2) Set.empty[Short] + else + settings.featuresSettings.supported + .filterNot(blockchainUpdater.approvedFeatures.keySet) + .filter(BlockchainFeatures.implemented) + .toSet + + log.debug(s"Adding ${unconfirmed.size} unconfirmed transaction(s) to new block") + Block.buildAndSign(version.toByte, currentTime, referencedBlockInfo.blockId, consensusData, unconfirmed, account, features) match { + case Left(e) => Left(e.err) + case Right(x) => Right((estimators, x, updatedMdConstraint.constraints.head)) + } + } + } yield block + ) + } private def generateOneMicroBlockTask(account: PrivateKeyAccount, accumulatedBlock: Block, @@ -242,16 +258,33 @@ class MinerImpl(allChannels: ChannelGroup, } } + private def nextBlockGenerationTime(fs: FunctionalitySettings, + height: Int, + block: Block, + account: PublicKeyAccount): Either[String, (Long, Long)] = { + val balance = GeneratingBalanceProvider.balance(blockchainUpdater, fs, height, account.toAddress) + if (GeneratingBalanceProvider.isMiningAllowed(blockchainUpdater, height, balance)) { + val cData = block.consensusData + val blockGS = cData.generationSignature.arr + val baseTarget = cData.baseTarget + val calculatedTs = pos.validBlockDelay(blockGS, account.publicKey, baseTarget, balance) + block.timestamp + if (0 < calculatedTs && calculatedTs < Long.MaxValue) { + Right((balance, calculatedTs)) + } else + Left(s"Invalid next block generation time: $calculatedTs") + } else Left(s"Balance $balance of ${account.address} is lower than required for generation") + } + private def generateBlockTask(account: PrivateKeyAccount): Task[Unit] = { { - val height = blockchainUpdater.height - val lastBlock = blockchainUpdater.lastBlock.get + val height = blockchainUpdater.height + val blockForHit = (blockchainUpdater.blockAt(height - 100) orElse blockchainUpdater.lastBlock).get for { _ <- checkAge(height, blockchainUpdater.lastBlockTimestamp.get) _ <- Either.cond(blockchainUpdater.accountScript(account).isEmpty, (), s"Account(${account.toAddress}) is scripted and therefore not allowed to forge blocks") - balanceAndTs <- nextBlockGenerationTime(height, blockchainUpdater, blockchainSettings.functionalitySettings, lastBlock, account) + balanceAndTs <- nextBlockGenerationTime(blockchainSettings.functionalitySettings, height, blockForHit, account) (balance, ts) = balanceAndTs offset = calcOffset(timeService, ts, minerSettings.minimalBlockGenerationOffset) } yield (offset, balance) @@ -261,7 +294,7 @@ class MinerImpl(allChannels: ChannelGroup, nextBlockGenerationTimes += account.toAddress -> (System.currentTimeMillis() + offset.toMillis) generateOneBlockTask(account, balance)(offset).flatMap { case Right((estimators, block, totalConstraint)) => - BlockAppender(checkpoint, blockchainUpdater, timeService, utx, settings, appenderScheduler)(block) + BlockAppender(checkpoint, blockchainUpdater, timeService, utx, pos, settings, appenderScheduler)(block) .asyncBoundary(minerScheduler) .map { case Left(err) => log.warn("Error mining Block: " + err.toString) diff --git a/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index 7ed8c8b4cc0..d1f0f62face 100644 --- a/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -1,6 +1,7 @@ package com.wavesplatform.state.appender import cats.data.EitherT +import com.wavesplatform.consensus.PoSCalculator import com.wavesplatform.metrics._ import com.wavesplatform.mining.Miner import com.wavesplatform.network._ @@ -25,6 +26,7 @@ object BlockAppender extends ScorexLogging with Instrumented { blockchainUpdater: BlockchainUpdater with Blockchain, time: Time, utxStorage: UtxPool, + pos: PoSCalculator, settings: WavesSettings, scheduler: Scheduler)(newBlock: Block): Task[Either[ValidationError, Option[BigInt]]] = Task { @@ -37,7 +39,7 @@ object BlockAppender extends ScorexLogging with Instrumented { (), BlockAppendError("Irrelevant block", newBlock)) _ = log.debug(s"Appending $newBlock") - maybeBaseHeight <- appendBlock(checkpoint, blockchainUpdater, utxStorage, time, settings)(newBlock) + maybeBaseHeight <- appendBlock(checkpoint, blockchainUpdater, utxStorage, pos, time, settings)(newBlock) } yield maybeBaseHeight map (_ => blockchainUpdater.score) } ) @@ -47,6 +49,7 @@ object BlockAppender extends ScorexLogging with Instrumented { blockchainUpdater: BlockchainUpdater with Blockchain, time: Time, utxStorage: UtxPool, + pos: PoSCalculator, settings: WavesSettings, allChannels: ChannelGroup, peerDatabase: PeerDatabase, @@ -56,7 +59,7 @@ object BlockAppender extends ScorexLogging with Instrumented { blockReceivingLag.safeRecord(System.currentTimeMillis() - newBlock.timestamp) (for { _ <- EitherT(Task.now(newBlock.signaturesValid())) - validApplication <- EitherT(apply(checkpoint, blockchainUpdater, time, utxStorage, settings, scheduler)(newBlock)) + validApplication <- EitherT(apply(checkpoint, blockchainUpdater, time, utxStorage, pos, settings, scheduler)(newBlock)) } yield validApplication).value.map { case Right(None) => // block already appended case Right(Some(_)) => diff --git a/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala b/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala index f41fdeede8d..52309b5f532 100644 --- a/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala +++ b/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala @@ -1,5 +1,6 @@ package com.wavesplatform.state.appender +import com.wavesplatform.consensus.PoSCalculator import com.wavesplatform.metrics.{BlockStats, Instrumented, Metrics} import com.wavesplatform.mining.Miner import com.wavesplatform.network.{InvalidBlockStorage, PeerDatabase, formatBlocks, id} @@ -23,6 +24,7 @@ object ExtensionAppender extends ScorexLogging with Instrumented { def apply(checkpoint: CheckpointService, blockchainUpdater: BlockchainUpdater with Blockchain, utxStorage: UtxPool, + pos: PoSCalculator, time: Time, settings: WavesSettings, invalidBlocks: InvalidBlockStorage, @@ -43,9 +45,10 @@ object ExtensionAppender extends ScorexLogging with Instrumented { val forkApplicationResultEi = Coeval { extension.view .map { b => - b -> appendBlock(checkpoint, blockchainUpdater, utxStorage, time, settings)(b).right.map { - _.foreach(bh => BlockStats.applied(b, BlockStats.Source.Ext, bh)) - } + b -> appendBlock(checkpoint, blockchainUpdater, utxStorage, pos, time, settings)(b).right + .map { + _.foreach(bh => BlockStats.applied(b, BlockStats.Source.Ext, bh)) + } } .zipWithIndex .collectFirst { case ((b, Left(e)), i) => (i, b, e) } diff --git a/src/main/scala/com/wavesplatform/state/appender/package.scala b/src/main/scala/com/wavesplatform/state/appender/package.scala index ab5cea3cb91..0c667a2cd22 100644 --- a/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -1,17 +1,15 @@ package com.wavesplatform.state -import com.wavesplatform.features.BlockchainFeatures -import com.wavesplatform.features.FeatureProvider._ +import com.wavesplatform.consensus.{GeneratingBalanceProvider, PoSCalculator} import com.wavesplatform.mining._ import com.wavesplatform.network._ -import com.wavesplatform.settings.{FunctionalitySettings, WavesSettings} +import com.wavesplatform.settings.WavesSettings import com.wavesplatform.utx.UtxPool import io.netty.channel.Channel import io.netty.channel.group.ChannelGroup import monix.eval.Task import scorex.block.Block import scorex.consensus.TransactionsOrdering -import scorex.transaction.PoSCalc._ import scorex.transaction.ValidationError.{BlockAppendError, BlockFromFuture, GenericError} import scorex.transaction._ import scorex.utils.{ScorexLogging, Time} @@ -49,20 +47,10 @@ package object appender extends ScorexLogging { } } - private def validateEffectiveBalance(blockchain: Blockchain, fs: FunctionalitySettings, block: Block, baseHeight: Int)( - effectiveBalance: Long): Either[String, Long] = - Either.cond( - block.timestamp < fs.minimalGeneratingBalanceAfter || - (block.timestamp >= fs.minimalGeneratingBalanceAfter && effectiveBalance >= MinimalEffectiveBalanceForGenerator1) || - blockchain.featureActivationHeight(BlockchainFeatures.SmallerMinimalGeneratingBalance.id).exists(baseHeight >= _) - && effectiveBalance >= MinimalEffectiveBalanceForGenerator2, - effectiveBalance, - s"generator's effective balance $effectiveBalance is less that required for generation" - ) - private[appender] def appendBlock(checkpoint: CheckpointService, blockchainUpdater: BlockchainUpdater with Blockchain, utxStorage: UtxPool, + pos: PoSCalculator, time: Time, settings: WavesSettings)(block: Block): Either[ValidationError, Option[Int]] = for { @@ -76,9 +64,17 @@ package object appender extends ScorexLogging { (), BlockAppendError(s"Account(${block.sender.toAddress}) is scripted are therefore not allowed to forge blocks", block) ) - _ <- blockConsensusValidation(blockchainUpdater, settings, time.correctedTime(), block) { height => - val balance = PoSCalc.generatingBalance(blockchainUpdater, settings.blockchainSettings.functionalitySettings, block.sender, height) - validateEffectiveBalance(blockchainUpdater, settings.blockchainSettings.functionalitySettings, block, height)(balance) + _ <- blockConsensusValidation(blockchainUpdater, settings, pos, time.correctedTime(), block) { height => + val balance = GeneratingBalanceProvider.balance(blockchainUpdater, settings.blockchainSettings.functionalitySettings, height, block.sender) + Either.cond( + GeneratingBalanceProvider.isEffectiveBalanceValid(blockchainUpdater, + settings.blockchainSettings.functionalitySettings, + height, + block, + balance), + balance, + s"generator's effective balance $balance is less that required for generation" + ) } baseHeight = blockchainUpdater.height maybeDiscardedTxs <- blockchainUpdater.processBlock(block) @@ -90,7 +86,7 @@ package object appender extends ScorexLogging { maybeDiscardedTxs.map(_ => baseHeight) } - private def blockConsensusValidation(blockchain: Blockchain, settings: WavesSettings, currentTs: Long, block: Block)( + private def blockConsensusValidation(blockchain: Blockchain, settings: WavesSettings, pos: PoSCalculator, currentTs: Long, block: Block)( genBalance: Int => Either[String, Long]): Either[ValidationError, Unit] = { val bcs = settings.blockchainSettings @@ -119,7 +115,7 @@ package object appender extends ScorexLogging { prevBlockData = parent.consensusData blockData = block.consensusData ggp = blockchain.parent(parent, 2) - cbt = calcBaseTarget(bcs.genesisSettings.averageBlockDelay, + cbt = pos.baseTarget(bcs.genesisSettings.averageBlockDelay.toSeconds, height, parent.consensusData.baseTarget, parent.timestamp, @@ -127,7 +123,7 @@ package object appender extends ScorexLogging { blockTime) bbt = blockData.baseTarget _ <- Either.cond(cbt == bbt, (), GenericError(s"declared baseTarget $bbt does not match calculated baseTarget $cbt")) - calcGs = calcGeneratorSignature(prevBlockData, generator) + calcGs = pos.generatorSignature(prevBlockData.generationSignature.arr, generator.publicKey) blockGs = blockData.generationSignature.arr _ <- Either.cond( calcGs.sameElements(blockGs), @@ -136,12 +132,12 @@ package object appender extends ScorexLogging { s"declared generation signature ${blockData.generationSignature.base58} does not match calculated generation signature ${ByteStr(calcGs).base58}") ) effectiveBalance <- genBalance(height).left.map(GenericError(_)) - hit = calcHit(prevBlockData, generator) - target = calcTarget(parent.timestamp, parent.consensusData.baseTarget, blockTime, effectiveBalance) + minValidBlockTime = parent.timestamp + pos.validBlockDelay(blockGs, parent.consensusData.baseTarget, effectiveBalance) _ <- Either.cond( - hit < target || (height == height1 && block.uniqueId == correctBlockId1) || (height == height2 && block.uniqueId == correctBlockId2), + blockTime > minValidBlockTime + || (height == height1 && block.uniqueId == correctBlockId1) || (height == height2 && block.uniqueId == correctBlockId2), (), - GenericError(s"calculated hit $hit >= calculated target $target") + GenericError(s"calculated time $minValidBlockTime < block time $blockTime") ) } yield () diff --git a/src/main/scala/scorex/api/http/AddressApiRoute.scala b/src/main/scala/scorex/api/http/AddressApiRoute.scala index e15b181232c..91f1e03ae03 100644 --- a/src/main/scala/scorex/api/http/AddressApiRoute.scala +++ b/src/main/scala/scorex/api/http/AddressApiRoute.scala @@ -4,6 +4,7 @@ import java.nio.charset.StandardCharsets import akka.http.scaladsl.marshalling.ToResponseMarshallable import akka.http.scaladsl.server.Route +import com.wavesplatform.consensus.GeneratingBalanceProvider import com.wavesplatform.crypto import com.wavesplatform.settings.{FunctionalitySettings, RestAPISettings} import com.wavesplatform.state.Blockchain @@ -16,9 +17,9 @@ import play.api.libs.json._ import scorex.BroadcastRoute import scorex.account.{Address, PublicKeyAccount} import scorex.crypto.encode.Base58 +import scorex.transaction.{TransactionFactory, ValidationError} import scorex.transaction.ValidationError.GenericError import scorex.transaction.smart.script.ScriptCompiler -import scorex.transaction.{PoSCalc, TransactionFactory, ValidationError} import scorex.utils.Time import scorex.wallet.Wallet @@ -358,7 +359,7 @@ case class AddressApiRoute(settings: RestAPISettings, BalanceDetails( account.address, portfolio.balance, - PoSCalc.generatingBalance(blockchain, functionalitySettings, account, blockchain.height), + GeneratingBalanceProvider.balance(blockchain, functionalitySettings, blockchain.height, account), portfolio.balance - portfolio.lease.out, portfolio.effectiveBalance ) diff --git a/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala b/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala index 39fd1e5185e..b525a637b18 100755 --- a/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala +++ b/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala @@ -1,6 +1,7 @@ package scorex.consensus.nxt.api.http import akka.http.scaladsl.server.Route +import com.wavesplatform.consensus.GeneratingBalanceProvider import com.wavesplatform.settings.{FunctionalitySettings, RestAPISettings} import com.wavesplatform.state.Blockchain import io.swagger.annotations._ @@ -8,7 +9,6 @@ import javax.ws.rs.Path import play.api.libs.json.Json import scorex.account.Address import scorex.api.http.{ApiRoute, CommonApiFunctions, InvalidAddress} -import scorex.transaction.PoSCalc @Path("/consensus") @Api(value = "/consensus") @@ -31,8 +31,7 @@ case class NxtConsensusApiRoute(settings: RestAPISettings, blockchain: Blockchai Address.fromString(address) match { case Left(_) => complete(InvalidAddress) case Right(account) => - val b = blockchain - complete(Json.obj("address" -> account.address, "balance" -> PoSCalc.generatingBalance(b, fs, account, b.height))) + complete(Json.obj("address" -> account.address, "balance" -> GeneratingBalanceProvider.balance(blockchain, fs, blockchain.height, account))) } } diff --git a/src/main/scala/scorex/transaction/PoSCalc.scala b/src/main/scala/scorex/transaction/PoSCalc.scala deleted file mode 100644 index 0912fc19d8e..00000000000 --- a/src/main/scala/scorex/transaction/PoSCalc.scala +++ /dev/null @@ -1,109 +0,0 @@ -package scorex.transaction - -import com.wavesplatform.crypto -import com.wavesplatform.features.BlockchainFeatures -import com.wavesplatform.features.FeatureProvider._ -import com.wavesplatform.settings.FunctionalitySettings -import com.wavesplatform.state.Blockchain -import scorex.account.{Address, PublicKeyAccount} -import scorex.block.Block -import scorex.consensus.nxt.NxtLikeConsensusBlockData -import scorex.utils.ScorexLogging - -import scala.concurrent.duration.FiniteDuration - -object PoSCalc extends ScorexLogging { - - val MinimalEffectiveBalanceForGenerator1: Long = 1000000000000L - val MinimalEffectiveBalanceForGenerator2: Long = 100000000000L - - private val AvgBlockTimeDepth: Int = 3 - - // Min BaseTarget value is 9 because only in this case it is possible to get to next integer value (10) - // then increasing base target by 11% and casting it to Long afterward (see lines 55 and 59) - private val MinBaseTarget: Long = 9 - - private val MinBlockDelaySeconds = 53 - private val MaxBlockDelaySeconds = 67 - private val BaseTargetGamma = 64 - - def calcTarget(prevBlockTimestamp: Long, prevBlockBaseTarget: Long, timestamp: Long, balance: Long): BigInt = { - val eta = (timestamp - prevBlockTimestamp) / 1000 - BigInt(prevBlockBaseTarget) * eta * balance - } - - def calcHit(lastBlockData: NxtLikeConsensusBlockData, generator: PublicKeyAccount): BigInt = - BigInt(1, calcGeneratorSignature(lastBlockData, generator).take(8).reverse) - - def calcGeneratorSignature(lastBlockData: NxtLikeConsensusBlockData, generator: PublicKeyAccount): Array[Byte] = - crypto.fastHash(lastBlockData.generationSignature.arr ++ generator.publicKey) - - def calcBaseTarget(avgBlockDelay: FiniteDuration, - parentHeight: Int, - parentBaseTarget: Long, - parentTimestamp: Long, - maybeGreatGrandParentTimestamp: Option[Long], - timestamp: Long): Long = { - val avgDelayInSeconds = avgBlockDelay.toSeconds - - val prevBaseTarget = parentBaseTarget - if (parentHeight % 2 == 0) { - val blocktimeAverage = maybeGreatGrandParentTimestamp.fold(timestamp - parentTimestamp)(ggpts => (timestamp - ggpts) / AvgBlockTimeDepth) / 1000 - val minBlocktimeLimit = normalize(MinBlockDelaySeconds, avgDelayInSeconds) - val maxBlocktimeLimit = normalize(MaxBlockDelaySeconds, avgDelayInSeconds) - val baseTargetGamma = normalize(BaseTargetGamma, avgDelayInSeconds) - - val baseTarget = (if (blocktimeAverage > avgDelayInSeconds) { - prevBaseTarget * Math.min(blocktimeAverage, maxBlocktimeLimit) / avgDelayInSeconds - } else { - prevBaseTarget - prevBaseTarget * baseTargetGamma * - (avgDelayInSeconds - Math.max(blocktimeAverage, minBlocktimeLimit)) / (avgDelayInSeconds * 100) - }).toLong - - normalizeBaseTarget(baseTarget, avgDelayInSeconds) - } else { - prevBaseTarget - } - } - - def generatingBalance(blockchain: Blockchain, fs: FunctionalitySettings, account: Address, atHeight: Int): Long = { - val generatingBalanceDepth = fs.generatingBalanceDepth(atHeight) - blockchain.effectiveBalance(account, atHeight, generatingBalanceDepth) - } - - def nextBlockGenerationTime(height: Int, - blockchain: Blockchain, - fs: FunctionalitySettings, - block: Block, - account: PublicKeyAccount): Either[String, (Long, Long)] = { - val balance = generatingBalance(blockchain, fs, account, height) - Either - .cond( - (!blockchain - .isFeatureActivated(BlockchainFeatures.SmallerMinimalGeneratingBalance, height) && balance >= MinimalEffectiveBalanceForGenerator1) || - (blockchain - .isFeatureActivated(BlockchainFeatures.SmallerMinimalGeneratingBalance, height) && balance >= MinimalEffectiveBalanceForGenerator2), - balance, - s"Balance $balance of ${account.address} is lower than required for generation" - ) - .flatMap { _ => - val cData = block.consensusData - val hit = calcHit(cData, account) - val t = cData.baseTarget - val calculatedTs = (hit * 1000) / (BigInt(t) * balance) + block.timestamp - if (0 < calculatedTs && calculatedTs < Long.MaxValue) { - Right((balance, calculatedTs.toLong)) - } else { - Left(s"Invalid next block generation time: $calculatedTs") - } - } - } - - private def normalizeBaseTarget(bt: Long, averageBlockDelaySeconds: Long): Long = { - val maxBaseTarget = Long.MaxValue / averageBlockDelaySeconds - if (bt < MinBaseTarget) MinBaseTarget else if (bt > maxBaseTarget) maxBaseTarget else bt - } - - private def normalize(value: Long, averageBlockDelaySeconds: Long): Double = value * averageBlockDelaySeconds / (60: Double) - -} diff --git a/src/test/resources/genesis.dev.conf b/src/test/resources/genesis.dev.conf new file mode 100644 index 00000000000..2f7af5b1f56 --- /dev/null +++ b/src/test/resources/genesis.dev.conf @@ -0,0 +1,21 @@ +# Configuration for genesis block generator +# To generate run from SBT: test:run src/test/resources/genesis.it.conf +genesis-generator { + network-type: "D" + + initial-balance: 10000000000000000 + base-target: 100 + average-block-delay: 60s + timestamp: 1489352400000 # Comment to use the current time + + # seed text -> share + # the sum of shares should be <= initial-balance + distributions { + "create genesis wallet devnet-0": 200000000000000 + "create genesis wallet devnet-0-d": 500000000000000 + "create genesis wallet devnet-1": 1000000000000000 + "create genesis wallet devnet-1-d": 1500000000000000 + "create genesis wallet devnet-2": 2000000000000000 + "create genesis wallet devnet-2-d": 2500000000000000 + } +} diff --git a/src/test/resources/genesis.it.conf b/src/test/resources/genesis.it.conf index 0cd33f6b61f..b0b2368dd2c 100644 --- a/src/test/resources/genesis.it.conf +++ b/src/test/resources/genesis.it.conf @@ -22,4 +22,4 @@ genesis-generator { "node09": 60000000000000 "node10": 830500000000000 } -} +} \ No newline at end of file diff --git a/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala b/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala new file mode 100644 index 00000000000..35c63e8e680 --- /dev/null +++ b/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala @@ -0,0 +1,72 @@ +package com.wavesplatform.consensus + +import org.scalatest.{Matchers, PropSpec} +import scala.util.Random + +class FairPoSCalculatorTest extends PropSpec with Matchers { + + val pos = FairPoSCalculator + + case class Block(height: Int, baseTarget: Long, timestamp: Long, delay: Long) + + def generationSignature: Array[Byte] = { + val arr = new Array[Byte](32) + Random.nextBytes(arr) + arr + } + + property("Correct consensus parameters of blocks generated with FairPoS") { + + val balance = 50000000L * 100000000L + + val blockDelaySeconds = 60 + + val defaultBaseTarget = 100L + + val first = Block(0, defaultBaseTarget, System.currentTimeMillis(), 0) + + val chain = (1 to 10000 foldLeft List(first))((acc, _) => { + acc match { + case last :: _ => + val delay = pos.validBlockDelay(generationSignature, last.baseTarget, balance) + val bt = pos.baseTarget( + blockDelaySeconds, + last.height + 1, + last.baseTarget, + last.timestamp, + if (acc.isDefinedAt(2)) Some(acc(2).timestamp) else None, + last.timestamp + delay + ) + + Block( + last.height + 1, + bt, + last.timestamp + delay, + delay + ) :: acc + + case _ => ??? + } + + }).reverse + + val maxBT = chain.maxBy(_.baseTarget).baseTarget + val avgBT = chain.map(_.baseTarget).sum / chain.length + val minBT = chain.minBy(_.baseTarget).baseTarget + + val maxDelay = chain.tail.maxBy(_.delay).delay + val avgDelay = chain.tail.map(_.delay).sum / (chain.length - 1) + val minDelay = chain.tail.minBy(_.delay).delay + + print( + s""" + |BT: $minBT $avgBT $maxBT + |Delay: $minDelay $avgDelay $maxDelay + """.stripMargin + ) + + assert(avgDelay < 80000 && avgDelay > 40000) + assert(avgBT < 200 && avgBT > 20) + } + +} diff --git a/waves-devnet.conf b/waves-devnet.conf index 2af84dceab1..39aa3eb5cec 100644 --- a/waves-devnet.conf +++ b/waves-devnet.conf @@ -27,17 +27,17 @@ waves { genesis { average-block-delay: 60000ms initial-base-target: 100 - timestamp: 1500635421931 - block-timestamp: 1500635421931 - signature: "GxifkzjW43Cg3xjpNjk5EwhVh5q9EN82WQpGMfNY33c1eCmLFtZGHARwRZLaSZaYss7iTt9yavTBWcXqagCBXii" + timestamp: 1489352400000 + block-timestamp: 1489352400000 + signature: "4fKa1vwqnuAoUEX2LJGaKKYZzQC3KYT3PDwash33RDP21GawXkFPJJKG6joScodQE5UXa5mymB6bomYFHpdS4YT4" initial-balance: 10000000000000000 transactions = [ - {recipient: "3FR3MyuXumwBj1bLC8xnW38iHtwm9Ugdm8K", amount: 1500000000000000}, - {recipient: "3FgScYB6MNdnN8m4xXddQe1Bjkwmd3U7YtM", amount: 1500000000000000}, - {recipient: "3FWXhvWq2r8m54MmCEZ3YZkLg2qUdGWbU3V", amount: 1500000000000000}, - {recipient: "3FkBWsgT9T3snZ4ZpzzQCJWQngJBLdDEPfU", amount: 1500000000000000}, + {recipient: "3FR3MyuXumwBj1bLC8xnW38iHtwm9Ugdm8K", amount: 2500000000000000}, + {recipient: "3FgScYB6MNdnN8m4xXddQe1Bjkwmd3U7YtM", amount: 200000000000000}, + {recipient: "3FWXhvWq2r8m54MmCEZ3YZkLg2qUdGWbU3V", amount: 1000000000000000}, + {recipient: "3FkBWsgT9T3snZ4ZpzzQCJWQngJBLdDEPfU", amount: 500000000000000}, {recipient: "3FeeqPbaEUQ8h3eQ4ZX9WcqzqskGbfTqM2a", amount: 1500000000000000}, - {recipient: "3FcSgww3tKZ7feQVmcnPFmRxsjqBodYz63x", amount: 1500000000000000} + {recipient: "3FcSgww3tKZ7feQVmcnPFmRxsjqBodYz63x", amount: 2000000000000000} ] } } From 24df14ddfc6f9f19745b286b904d104de261c282 Mon Sep 17 00:00:00 2001 From: Alexandr Date: Fri, 18 May 2018 02:38:41 +0300 Subject: [PATCH 05/52] NODE-129: Take GenerationSignature from block at height = current - 100 --- .../com/wavesplatform/mining/Miner.scala | 46 ++++++++++--------- .../state/appender/package.scala | 40 ++++++++-------- 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/src/main/scala/com/wavesplatform/mining/Miner.scala b/src/main/scala/com/wavesplatform/mining/Miner.scala index 35cf37a137a..fa38c591a38 100644 --- a/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -62,7 +62,7 @@ class MinerImpl(allChannels: ChannelGroup, pos: PoSCalculator, val minerScheduler: SchedulerService, val appenderScheduler: SchedulerService) - extends Miner + extends Miner with MinerDebugInfo with ScorexLogging with Instrumented { @@ -96,12 +96,12 @@ class MinerImpl(allChannels: ChannelGroup, blockAge <= minerSettings.intervalAfterLastBlockThenGenerationIsAllowed, (), s"BlockChain is too old (last block timestamp is $parentTimestamp generated $blockAge ago)" - )) + )) private def ngEnabled: Boolean = blockchainUpdater.featureActivationHeight(BlockchainFeatures.NG.id).exists(blockchainUpdater.height > _ + 1) private def generateOneBlockTask(account: PrivateKeyAccount, balance: Long)( - delay: FiniteDuration): Task[Either[String, (MiningConstraints, Block, MiningConstraint)]] = { + delay: FiniteDuration): Task[Either[String, (MiningConstraints, Block, MiningConstraint)]] = { Task { forgeBlock(account, balance) }.delayExecution(delay) @@ -113,12 +113,16 @@ class MinerImpl(allChannels: ChannelGroup, val version = if (height <= blockchainSettings.functionalitySettings.blockVersion3AfterHeight) PlainBlockVersion else NgBlockVersion val lastBlock = blockchainUpdater.lastBlock.get val greatGrandParentTimestamp = blockchainUpdater.parent(lastBlock, 2).map(_.timestamp) - val referencedBlockInfo = blockchainUpdater.bestLastBlockInfo(System.currentTimeMillis() - minMicroBlockDurationMills).get - val pc = allChannels.size() - lazy val currentTime = timeService.correctedTime() - lazy val blockDelay = currentTime - lastBlock.timestamp + val generationSignature = blockchainUpdater + .blockAt(height - 100) + .map(_.consensusData.generationSignature.arr) + .getOrElse(lastBlock.consensusData.generationSignature.arr) + val referencedBlockInfo = blockchainUpdater.bestLastBlockInfo(System.currentTimeMillis() - minMicroBlockDurationMills).get + val pc = allChannels.size() + lazy val currentTime = timeService.correctedTime() + lazy val blockDelay = currentTime - lastBlock.timestamp lazy val validBlockDelay = pos.validBlockDelay( - referencedBlockInfo.consensus.generationSignature.arr, + generationSignature, account.publicKey, referencedBlockInfo.consensus.baseTarget, balance @@ -127,8 +131,8 @@ class MinerImpl(allChannels: ChannelGroup, blockBuildTimeStats, for { _ <- Either.cond(pc >= minerSettings.quorum, - (), - s"Quorum not available ($pc/${minerSettings.quorum}, not forging block with ${account.address}") + (), + s"Quorum not available ($pc/${minerSettings.quorum}, not forging block with ${account.address}") _ <- Either.cond( blockDelay > validBlockDelay, (), @@ -140,12 +144,12 @@ class MinerImpl(allChannels: ChannelGroup, block <- { val avgBlockDelay = blockchainSettings.genesisSettings.averageBlockDelay val btg = pos.baseTarget(avgBlockDelay.toSeconds, - height, - referencedBlockInfo.consensus.baseTarget, - referencedBlockInfo.timestamp, - greatGrandParentTimestamp, - currentTime) - val gs = pos.generatorSignature(referencedBlockInfo.consensus.generationSignature.arr, account.publicKey) + height, + referencedBlockInfo.consensus.baseTarget, + referencedBlockInfo.timestamp, + greatGrandParentTimestamp, + currentTime) + val gs = pos.generatorSignature(generationSignature, account.publicKey) val consensusData = NxtLikeConsensusBlockData(btg, ByteStr(gs)) val sortInBlock = blockchainUpdater.height <= blockchainSettings.functionalitySettings.dontRequireSortedTransactionsAfter @@ -277,14 +281,14 @@ class MinerImpl(allChannels: ChannelGroup, private def generateBlockTask(account: PrivateKeyAccount): Task[Unit] = { { - val height = blockchainUpdater.height - val blockForHit = (blockchainUpdater.blockAt(height - 100) orElse blockchainUpdater.lastBlock).get + val height = blockchainUpdater.height + val lastBlock = blockchainUpdater.lastBlock.get for { _ <- checkAge(height, blockchainUpdater.lastBlockTimestamp.get) _ <- Either.cond(blockchainUpdater.accountScript(account).isEmpty, - (), - s"Account(${account.toAddress}) is scripted and therefore not allowed to forge blocks") - balanceAndTs <- nextBlockGenerationTime(blockchainSettings.functionalitySettings, height, blockForHit, account) + (), + s"Account(${account.toAddress}) is scripted and therefore not allowed to forge blocks") + balanceAndTs <- nextBlockGenerationTime(blockchainSettings.functionalitySettings, height, lastBlock, account) (balance, ts) = balanceAndTs offset = calcOffset(timeService, ts, minerSettings.minimalBlockGenerationOffset) } yield (offset, balance) diff --git a/src/main/scala/com/wavesplatform/state/appender/package.scala b/src/main/scala/com/wavesplatform/state/appender/package.scala index 0c667a2cd22..3d9b5122017 100644 --- a/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -26,13 +26,13 @@ package object appender extends ScorexLogging { private val height2 = 813207 private[appender] def processAndBlacklistOnFailure[A, B]( - ch: Channel, - peerDatabase: PeerDatabase, - miner: Miner, - allChannels: ChannelGroup, - start: => String, - success: => String, - errorPrefix: String)(f: => Task[Either[B, Option[BigInt]]]): Task[Either[B, Option[BigInt]]] = { + ch: Channel, + peerDatabase: PeerDatabase, + miner: Miner, + allChannels: ChannelGroup, + start: => String, + success: => String, + errorPrefix: String)(f: => Task[Either[B, Option[BigInt]]]): Task[Either[B, Option[BigInt]]] = { log.debug(start) f map { @@ -68,10 +68,10 @@ package object appender extends ScorexLogging { val balance = GeneratingBalanceProvider.balance(blockchainUpdater, settings.blockchainSettings.functionalitySettings, height, block.sender) Either.cond( GeneratingBalanceProvider.isEffectiveBalanceValid(blockchainUpdater, - settings.blockchainSettings.functionalitySettings, - height, - block, - balance), + settings.blockchainSettings.functionalitySettings, + height, + block, + balance), balance, s"generator's effective balance $balance is less that required for generation" ) @@ -87,12 +87,16 @@ package object appender extends ScorexLogging { } private def blockConsensusValidation(blockchain: Blockchain, settings: WavesSettings, pos: PoSCalculator, currentTs: Long, block: Block)( - genBalance: Int => Either[String, Long]): Either[ValidationError, Unit] = { + genBalance: Int => Either[String, Long]): Either[ValidationError, Unit] = { val bcs = settings.blockchainSettings val fs = bcs.functionalitySettings val blockTime = block.timestamp val generator = block.signerData.generator + val generatorSignature = + (blockchain.blockAt(blockchain.height - 100) orElse blockchain.lastBlock) + .map(_.consensusData.generationSignature.arr) + .get val r: Either[ValidationError, Unit] = for { height <- blockchain.heightOf(block.reference).toRight(GenericError(s"height: history does not contain parent ${block.reference}")) @@ -116,14 +120,14 @@ package object appender extends ScorexLogging { blockData = block.consensusData ggp = blockchain.parent(parent, 2) cbt = pos.baseTarget(bcs.genesisSettings.averageBlockDelay.toSeconds, - height, - parent.consensusData.baseTarget, - parent.timestamp, - ggp.map(_.timestamp), - blockTime) + height, + parent.consensusData.baseTarget, + parent.timestamp, + ggp.map(_.timestamp), + blockTime) bbt = blockData.baseTarget _ <- Either.cond(cbt == bbt, (), GenericError(s"declared baseTarget $bbt does not match calculated baseTarget $cbt")) - calcGs = pos.generatorSignature(prevBlockData.generationSignature.arr, generator.publicKey) + calcGs = pos.generatorSignature(generatorSignature, generator.publicKey) blockGs = blockData.generationSignature.arr _ <- Either.cond( calcGs.sameElements(blockGs), From b0e1087cd6fb1c0ccd07b7e2f4b7a44d936a687e Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Fri, 18 May 2018 20:03:55 +0300 Subject: [PATCH 06/52] NODE-732 unary operations ('-', '!') --- .../lang/v1/evaluator/ctx/impl/PureContext.scala | 16 +++++++++++++++- .../wavesplatform/lang/v1/parser/Parser.scala | 9 ++++++++- .../lang/v1/parser/UnaryOperation.scala | 11 +++++++++++ .../lang/v1/testing/ScriptGen.scala | 6 ++++-- 4 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala index ffb98d351d9..b04bd583573 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala @@ -67,6 +67,20 @@ object PureContext { case _ => ??? } + val uMinus = PredefFunction("-", 1, LONG, List("n" -> LONG)) { + case (n: Long) :: Nil => { + Right(-n) + } + case _ => ??? + } + + val uNot = PredefFunction("!", 1, BOOLEAN, List("p" -> BOOLEAN)) { + case (p: Boolean) :: Nil => { + Right(!p) + } + case _ => ??? + } + private def createTryOp(op: BinaryOperation, t: TYPE, r: TYPE)(body: (t.Underlying, t.Underlying) => r.Underlying) = { PredefFunction(opsToFunctions(op), 1, r, List("a" -> t, "b" -> t)) { case a :: b :: Nil => @@ -89,7 +103,7 @@ object PureContext { val ge = createOp(GE_OP, LONG, BOOLEAN)(_ >= _) val gt = createOp(GT_OP, LONG, BOOLEAN)(_ > _) - val operators: Seq[PredefFunction] = Seq(sumLong, sumString, sumByteVector, eqLong, eqByteVector, eqBool, eqString, ge, gt, getElement, getListSize) + val operators: Seq[PredefFunction] = Seq(sumLong, sumString, sumByteVector, eqLong, eqByteVector, eqBool, eqString, ge, gt, getElement, getListSize, uMinus, uNot) lazy val instance = EvaluationContext.build(types = Seq.empty, diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala index f98f153fb04..ed67621811e 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala @@ -2,6 +2,7 @@ package com.wavesplatform.lang.v1.parser import Expressions._ import BinaryOperation._ +import UnaryOperation._ import fastparse.{WhitespaceApi, core} import scodec.bits.ByteVector @@ -86,8 +87,14 @@ object Parser { private val atom = P(ifP | matchP | byteVectorP | stringP | numberP | trueP | falseP | block | maybeAccessP) + def unaryOp(ops: Seq[(P[Any], EXPR => EXPR)]): P[EXPR] = { + ops.foldRight(atom) { + (op, acc) => (op._1.map(_ => ()) ~ P(unaryOp(ops))).map(op._2) | acc + } + } + private def binaryOp(rest: List[(String, BinaryOperation)]): P[EXPR] = rest match { - case Nil => atom + case Nil => unaryOp(unaryOps) case (lessPriorityOp, kind) :: restOps => val operand = binaryOp(restOps) P(operand ~ (lessPriorityOp.!.map(_ => kind) ~ operand).rep()).map { diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala new file mode 100644 index 00000000000..31f346e4b70 --- /dev/null +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala @@ -0,0 +1,11 @@ +package com.wavesplatform.lang.v1.parser + +import Expressions._ +import fastparse.all._ + +object UnaryOperation { + val unaryOps = List( + P("-" ~ !CharIn('0' to '9')) -> {e: EXPR => FUNCTION_CALL("-", List(e))}, + P("!") -> {e: EXPR => FUNCTION_CALL("!", List(e))} + ) +} diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index ae2c7e9f935..8975db1a816 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -20,7 +20,7 @@ trait ScriptGen { if((BigInt(v1) + BigInt(v2)).isValidLong) } yield (BINARY_OP(i1, SUM_OP, i2), (v1 + v2)) - def INTGen(gas: Int): Gen[(EXPR, Long)] = if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), IF_INTgen(gas - 1)) else CONST_LONGgen + def INTGen(gas: Int): Gen[(EXPR, Long)] = if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), IF_INTgen(gas - 1), INTGen(gas-1).map(e => (FUNCTION_CALL("-",List(e._1)), -e._2))) else CONST_LONGgen def GEgen(gas: Int): Gen[(EXPR, Boolean)] = for { @@ -104,6 +104,8 @@ trait ScriptGen { case CONST_STRING(x) => withWhitespaces(s"""\"$x\"""") case TRUE => withWhitespaces("true") case FALSE => withWhitespaces("false") + case FUNCTION_CALL("-", List(CONST_LONG(v))) if (v>=0) => s"-($v)" + case FUNCTION_CALL(op, List(e)) => toString(e).map(e => s"$op$e") case BINARY_OP(x, op: BinaryOperation, y) => for { arg1 <- toString(x) @@ -126,7 +128,7 @@ trait ScriptGen { trait ScriptGenParser extends ScriptGen { override def BOOLgen(gas: Int): Gen[(EXPR, Boolean)] = { - if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1), REFgen.map(r => (r, false))) + if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1), REFgen.map(r => (r, false)), BOOLgen(gas-1).map(e => (FUNCTION_CALL("!", List(e._1)), !e._2))) else Gen.const((TRUE, true)) } From 0f07b6437e0940e5237799df94743f3868e40522 Mon Sep 17 00:00:00 2001 From: Alexandr M Date: Fri, 18 May 2018 20:26:10 +0300 Subject: [PATCH 07/52] NODE-129: PoSSelector, BlockConsensusValidation refactored --- .../wavesplatform/it/BaseTargetChecker.scala | 6 +- .../scala/com/wavesplatform/Application.scala | 2 +- .../scala/com/wavesplatform/Importer.scala | 2 +- .../consensus/PoSCalculator.scala | 30 +--- .../wavesplatform/consensus/PoSSelector.scala | 104 ++++++++++++ .../com/wavesplatform/mining/Miner.scala | 157 ++++++++++-------- .../state/appender/BlockAppender.scala | 6 +- .../state/appender/ExtensionAppender.scala | 4 +- .../state/appender/package.scala | 121 ++++++-------- .../nxt/api/http/NxtConsensusApiRoute.scala | 8 +- 10 files changed, 258 insertions(+), 182 deletions(-) create mode 100644 src/main/scala/com/wavesplatform/consensus/PoSSelector.scala diff --git a/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala b/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala index 77b51a5904c..e74b05f14de 100644 --- a/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala +++ b/it/src/main/scala/com/wavesplatform/it/BaseTargetChecker.scala @@ -25,7 +25,7 @@ object BaseTargetChecker { val genesisBlock = Block.genesis(settings.blockchainSettings.genesisSettings).explicitGet() val db = openDB("/tmp/tmp-db", 1024) val bu = StorageFactory(settings, db, NTP) - val pos = new PoSSelector(bu) + val pos = new PoSSelector(bu, settings.blockchainSettings) bu.processBlock(genesisBlock) println(s"Genesis TS = ${Instant.ofEpochMilli(genesisBlock.timestamp)}") @@ -36,7 +36,9 @@ object BaseTargetChecker { val address = account.toAddress val balance = bu.balance(address, None) val consensus = genesisBlock.consensusData - val timeDelay = pos.validBlockDelay(consensus.generationSignature.arr, account.publicKey, consensus.baseTarget, balance) + val timeDelay = pos + .getValidBlockDelay(bu.height, account.publicKey, consensus.baseTarget, balance) + .explicitGet() f"$address: ${timeDelay * 1e-3}%10.3f s" } diff --git a/src/main/scala/com/wavesplatform/Application.scala b/src/main/scala/com/wavesplatform/Application.scala index 0437750a8de..2dd194defe4 100644 --- a/src/main/scala/com/wavesplatform/Application.scala +++ b/src/main/scala/com/wavesplatform/Application.scala @@ -125,7 +125,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con val knownInvalidBlocks = new InvalidBlockStorageImpl(settings.synchronizationSettings.invalidBlocksStorage) - val pos = new PoSSelector(blockchainUpdater) + val pos = new PoSSelector(blockchainUpdater, settings.blockchainSettings) val miner = if (settings.minerSettings.enable) diff --git a/src/main/scala/com/wavesplatform/Importer.scala b/src/main/scala/com/wavesplatform/Importer.scala index f791c87c1fc..624fba99e75 100644 --- a/src/main/scala/com/wavesplatform/Importer.scala +++ b/src/main/scala/com/wavesplatform/Importer.scala @@ -58,7 +58,7 @@ object Importer extends ScorexLogging { case Success(inputStream) => val db = openDB(settings.dataDirectory, settings.levelDbCacheSize) val blockchainUpdater = StorageFactory(settings, db, NTP) - val pos = new PoSSelector(blockchainUpdater) + val pos = new PoSSelector(blockchainUpdater, settings.blockchainSettings) val checkpoint = new CheckpointServiceImpl(db, settings.checkpointsSettings) val extAppender = BlockAppender(checkpoint, blockchainUpdater, NTP, utxPoolStub, pos, settings, scheduler) _ checkGenesis(settings, blockchainUpdater) diff --git a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala index 5e438473d5e..a0d64fe7734 100644 --- a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala +++ b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala @@ -1,9 +1,6 @@ package com.wavesplatform.consensus import com.wavesplatform.crypto -import com.wavesplatform.features.BlockchainFeatures -import com.wavesplatform.features.FeatureProvider._ -import com.wavesplatform.state.Blockchain trait PoSCalculator { protected val HitSize: Int = 8 @@ -23,7 +20,7 @@ trait PoSCalculator { maybeGreatGrandParentTimestamp: Option[Long], timestamp: Long): Long - protected def hit(generatorSignature: Array[Byte]): BigInt = BigInt(1, generatorSignature.take(HitSize).reverse) + def hit(generatorSignature: Array[Byte]): BigInt = BigInt(1, generatorSignature.take(HitSize).reverse) def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long @@ -35,10 +32,10 @@ trait PoSCalculator { calculateDelay(hit(generatorSignature(genSig, publicKey)), baseTarget, balance) } - protected def normalize(value: Long, targetBlockDelaySeconds: Long): Double = + def normalize(value: Long, targetBlockDelaySeconds: Long): Double = value * targetBlockDelaySeconds / (60: Double) - protected def normalizeBaseTarget(baseTarget: Long, targetBlockDelaySeconds: Long): Long = { + def normalizeBaseTarget(baseTarget: Long, targetBlockDelaySeconds: Long): Long = { baseTarget .max(MinBaseTarget) .min(Long.MaxValue / targetBlockDelaySeconds) @@ -46,27 +43,6 @@ trait PoSCalculator { } -class PoSSelector(val blockchain: Blockchain) extends PoSCalculator { - - protected def pos: PoSCalculator = - if (fair(blockchain.height)) FairPoSCalculator - else NxtPoSCalculator - - override def baseTarget(targetBlockDelaySeconds: Long, - prevHeight: Int, - prevBaseTarget: Long, - parentTimestamp: Long, - maybeGreatGrandParentTimestamp: Option[Long], - timestamp: Long): Long = { - pos.baseTarget(targetBlockDelaySeconds, prevHeight, prevBaseTarget, parentTimestamp, maybeGreatGrandParentTimestamp, timestamp) - } - - def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long = - pos.calculateDelay(hit, bt, balance) - - private def fair(height: Int): Boolean = blockchain.activatedFeaturesAt(height).contains(BlockchainFeatures.FairPoS.id) -} - object NxtPoSCalculator extends PoSCalculator { protected val MinBlockDelaySeconds = 53 protected val MaxBlockDelaySeconds = 67 diff --git a/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala b/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala new file mode 100644 index 00000000000..814485aff49 --- /dev/null +++ b/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala @@ -0,0 +1,104 @@ +package com.wavesplatform.consensus + +import com.wavesplatform.features.BlockchainFeatures +import com.wavesplatform.features.FeatureProvider._ +import com.wavesplatform.settings.BlockchainSettings +import com.wavesplatform.state._ +import com.wavesplatform.state.{Blockchain, ByteStr} +import scorex.block.Block +import scorex.consensus.nxt.NxtLikeConsensusBlockData +import scorex.transaction.ValidationError +import scorex.transaction.ValidationError.GenericError +import cats.implicits._ +import scala.concurrent.duration.FiniteDuration + +class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { + + protected def pos(height: Int): PoSCalculator = + if (fairPosActivated(height)) FairPoSCalculator + else NxtPoSCalculator + + def consensusData(accountPublicKey: Array[Byte], + height: Int, + targetBlockDelay: FiniteDuration, + refBlockBT: Long, + refBlockTS: Long, + greatGrandParentTS: Option[Long], + currentTime: Long): Either[ValidationError, NxtLikeConsensusBlockData] = { + val bt = baseTarget(targetBlockDelay.toSeconds, height, refBlockBT, refBlockTS, greatGrandParentTS, currentTime) + getBlockForGS(height) + .map(_.consensusData.generationSignature.arr) + .map(gs => NxtLikeConsensusBlockData(bt, ByteStr(gs))) + .toRight(GenericError("No blocks in blockchain")) + } + + def getValidBlockDelay(height: Int, accountPublicKey: Array[Byte], refBlockBT: Long, balance: Long): Either[ValidationError, Long] = { + val pc = pos(height) + + getBlockForGS(height) + .map(_.consensusData.generationSignature.arr) + .map(pc.generatorSignature(_, accountPublicKey)) + .map(pc.hit) + .map(pc.calculateDelay(_, refBlockBT, balance)) + .toRight(GenericError("No blocks in blockchain")) + } + + def validateBlockDelay(height: Int, block: Block, parent: Block, effectiveBalance: Long): Either[ValidationError, Unit] = { + getValidBlockDelay(height, block.signerData.generator.publicKey, parent.consensusData.baseTarget, effectiveBalance) + .map(_ + parent.timestamp < block.timestamp) + .ensure(GenericError(s"Block time ${block.timestamp} less than expected"))(identity) + .map(_ => ()) + } + + def validateGeneratorSignature(height: Int, block: Block): Either[ValidationError, Unit] = { + getBlockForGS(height) + .map(_.consensusData.generationSignature.arr) + .toRight(GenericError("No blocks in blockchain T.T")) + .ensure(GenericError("Generation signatures doesnot match"))(_ sameElements block.consensusData.generationSignature.arr) + .map(_ => ()) + } + + def validateBaseTarget(height: Int, block: Block, parent: Block, grandParent: Option[Block]): Either[ValidationError, Unit] = { + val blockBT = block.consensusData.baseTarget + val blockTS = block.timestamp + + val expectedBT = baseTarget( + settings.genesisSettings.averageBlockDelay.toSeconds, + height, + parent.consensusData.baseTarget, + parent.timestamp, + grandParent.map(_.timestamp), + blockTS + ) + + Either.cond( + expectedBT == blockBT, + (), + GenericError(s"declared baseTarget $blockBT does not match calculated baseTarget $expectedBT") + ) + } + + private def getBlockForGS(height: Int): Option[Block] = { + if (fairPosActivated(height)) blockchain.blockAt(height - 100) orElse blockchain.lastBlock + else blockchain.lastBlock + } + + def baseTarget(targetBlockDelaySeconds: Long, + prevHeight: Int, + prevBaseTarget: Long, + parentTimestamp: Long, + maybeGreatGrandParentTimestamp: Option[Long], + timestamp: Long): Long = { + pos(blockchain.height) + .baseTarget( + targetBlockDelaySeconds, + prevHeight, + prevBaseTarget, + parentTimestamp, + maybeGreatGrandParentTimestamp, + timestamp + ) + } + + private def fairPosActivated(height: Int): Boolean = blockchain.activatedFeaturesAt(height).contains(BlockchainFeatures.FairPoS.id) +} diff --git a/src/main/scala/com/wavesplatform/mining/Miner.scala b/src/main/scala/com/wavesplatform/mining/Miner.scala index fa38c591a38..c08096bb3a6 100644 --- a/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -1,7 +1,8 @@ package com.wavesplatform.mining import cats.data.EitherT -import com.wavesplatform.consensus.{GeneratingBalanceProvider, PoSCalculator} +import cats.implicits._ +import com.wavesplatform.consensus.{GeneratingBalanceProvider, PoSSelector} import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.features.FeatureProvider._ import com.wavesplatform.metrics.{BlockStats, HistogramExt, Instrumented} @@ -59,10 +60,10 @@ class MinerImpl(allChannels: ChannelGroup, timeService: Time, utx: UtxPool, wallet: Wallet, - pos: PoSCalculator, + pos: PoSSelector, val minerScheduler: SchedulerService, val appenderScheduler: SchedulerService) - extends Miner + extends Miner with MinerDebugInfo with ScorexLogging with Instrumented { @@ -96,85 +97,93 @@ class MinerImpl(allChannels: ChannelGroup, blockAge <= minerSettings.intervalAfterLastBlockThenGenerationIsAllowed, (), s"BlockChain is too old (last block timestamp is $parentTimestamp generated $blockAge ago)" - )) + )) + + private def checkScript(account: PrivateKeyAccount): Either[String, Unit] = { + Either.cond(blockchainUpdater.accountScript(account).isEmpty, + (), + s"Account(${account.toAddress}) is scripted and therefore not allowed to forge blocks") + } private def ngEnabled: Boolean = blockchainUpdater.featureActivationHeight(BlockchainFeatures.NG.id).exists(blockchainUpdater.height > _ + 1) private def generateOneBlockTask(account: PrivateKeyAccount, balance: Long)( - delay: FiniteDuration): Task[Either[String, (MiningConstraints, Block, MiningConstraint)]] = { + delay: FiniteDuration): Task[Either[String, (MiningConstraints, Block, MiningConstraint)]] = { Task { forgeBlock(account, balance) }.delayExecution(delay) } + private def consensusData(height: Int, + account: PrivateKeyAccount, + lastBlock: Block, + refBlockBT: Long, + refBlockTS: Long, + balance: Long, + currentTime: Long): Either[String, NxtLikeConsensusBlockData] = { + pos + .consensusData( + account.publicKey, + height, + blockchainSettings.genesisSettings.averageBlockDelay, + refBlockBT, + refBlockTS, + blockchainUpdater.parent(lastBlock, 2).map(_.timestamp), + currentTime + ) + .leftMap(_.toString) + } + private def forgeBlock(account: PrivateKeyAccount, balance: Long): Either[String, (MiningConstraints, Block, MiningConstraint)] = { // should take last block right at the time of mining since microblocks might have been added - val height = blockchainUpdater.height - val version = if (height <= blockchainSettings.functionalitySettings.blockVersion3AfterHeight) PlainBlockVersion else NgBlockVersion - val lastBlock = blockchainUpdater.lastBlock.get - val greatGrandParentTimestamp = blockchainUpdater.parent(lastBlock, 2).map(_.timestamp) - val generationSignature = blockchainUpdater - .blockAt(height - 100) - .map(_.consensusData.generationSignature.arr) - .getOrElse(lastBlock.consensusData.generationSignature.arr) + val height = blockchainUpdater.height + val version = if (height <= blockchainSettings.functionalitySettings.blockVersion3AfterHeight) PlainBlockVersion else NgBlockVersion + val lastBlock = blockchainUpdater.lastBlock.get val referencedBlockInfo = blockchainUpdater.bestLastBlockInfo(System.currentTimeMillis() - minMicroBlockDurationMills).get - val pc = allChannels.size() + val refBlockBT = referencedBlockInfo.consensus.baseTarget + val refBlockTS = referencedBlockInfo.timestamp + val refBlockID = referencedBlockInfo.blockId lazy val currentTime = timeService.correctedTime() lazy val blockDelay = currentTime - lastBlock.timestamp - lazy val validBlockDelay = pos.validBlockDelay( - generationSignature, - account.publicKey, - referencedBlockInfo.consensus.baseTarget, - balance - ) measureSuccessful( blockBuildTimeStats, for { - _ <- Either.cond(pc >= minerSettings.quorum, - (), - s"Quorum not available ($pc/${minerSettings.quorum}, not forging block with ${account.address}") - _ <- Either.cond( - blockDelay > validBlockDelay, - (), - s"${System.currentTimeMillis()}: Block delay $blockDelay was NOT less than estimated delay $validBlockDelay, not forging block with ${account.address}" - ) + _ <- checkQuorumAvailable() + validBlockDelay <- pos + .getValidBlockDelay(height, account.publicKey, refBlockBT, balance) + .leftMap(_.toString) + .ensure(s"$currentTime: Block delay $blockDelay was NOT less than estimated delay")(_ < blockDelay) _ = log.debug( - s"Forging with ${account.address}, Time $blockDelay > Estimated Time $validBlockDelay, balance $balance, prev block ${referencedBlockInfo.blockId}") - _ = log.debug(s"Previous block ID ${referencedBlockInfo.blockId} at $height with target ${referencedBlockInfo.consensus.baseTarget}") - block <- { - val avgBlockDelay = blockchainSettings.genesisSettings.averageBlockDelay - val btg = pos.baseTarget(avgBlockDelay.toSeconds, - height, - referencedBlockInfo.consensus.baseTarget, - referencedBlockInfo.timestamp, - greatGrandParentTimestamp, - currentTime) - val gs = pos.generatorSignature(generationSignature, account.publicKey) - val consensusData = NxtLikeConsensusBlockData(btg, ByteStr(gs)) - val sortInBlock = blockchainUpdater.height <= blockchainSettings.functionalitySettings.dontRequireSortedTransactionsAfter - - val estimators = MiningConstraints(minerSettings, blockchainUpdater, height) - val mdConstraint = MultiDimensionalMiningConstraint(estimators.total, estimators.keyBlock) - val (unconfirmed, updatedMdConstraint) = utx.packUnconfirmed(mdConstraint, sortInBlock) - - val features = - if (version <= 2) Set.empty[Short] - else - settings.featuresSettings.supported - .filterNot(blockchainUpdater.approvedFeatures.keySet) - .filter(BlockchainFeatures.implemented) - .toSet - - log.debug(s"Adding ${unconfirmed.size} unconfirmed transaction(s) to new block") - Block.buildAndSign(version.toByte, currentTime, referencedBlockInfo.blockId, consensusData, unconfirmed, account, features) match { - case Left(e) => Left(e.err) - case Right(x) => Right((estimators, x, updatedMdConstraint.constraints.head)) - } - } - } yield block + s"Forging with ${account.address}, Time $blockDelay > Estimated Time $validBlockDelay, balance $balance, prev block $refBlockID") + _ = log.debug(s"Previous block ID $refBlockID at $height with target $refBlockBT") + consensusData <- consensusData(height, account, lastBlock, refBlockBT, refBlockTS, balance, currentTime) + estimators = MiningConstraints(minerSettings, blockchainUpdater, height) + mdConstraint = MultiDimensionalMiningConstraint(estimators.total, estimators.keyBlock) + (unconfirmed, updatedMdConstraint) = utx.packUnconfirmed(mdConstraint, isSortingRequired()) + _ = log.debug(s"Adding ${unconfirmed.size} unconfirmed transaction(s) to new block") + block <- Block + .buildAndSign(version.toByte, currentTime, refBlockID, consensusData, unconfirmed, account, blockFeatures(version)) + .leftMap(_.err) + } yield (estimators, block, updatedMdConstraint.constraints.head) ) } + private def checkQuorumAvailable(): Either[String, Unit] = { + val chanCount = allChannels.size() + Either.cond(chanCount >= minerSettings.quorum, (), s"Quorum not available ($chanCount/${minerSettings.quorum}, not forging block.") + } + + private def isSortingRequired(): Boolean = blockchainUpdater.height <= blockchainSettings.functionalitySettings.dontRequireSortedTransactionsAfter + + private def blockFeatures(version: Byte): Set[Short] = { + if (version <= 2) Set.empty[Short] + else + settings.featuresSettings.supported + .filterNot(blockchainUpdater.approvedFeatures.keySet) + .filter(BlockchainFeatures.implemented) + .toSet + } + private def generateOneMicroBlockTask(account: PrivateKeyAccount, accumulatedBlock: Block, constraints: MiningConstraints, @@ -267,15 +276,19 @@ class MinerImpl(allChannels: ChannelGroup, block: Block, account: PublicKeyAccount): Either[String, (Long, Long)] = { val balance = GeneratingBalanceProvider.balance(blockchainUpdater, fs, height, account.toAddress) + if (GeneratingBalanceProvider.isMiningAllowed(blockchainUpdater, height, balance)) { - val cData = block.consensusData - val blockGS = cData.generationSignature.arr - val baseTarget = cData.baseTarget - val calculatedTs = pos.validBlockDelay(blockGS, account.publicKey, baseTarget, balance) + block.timestamp - if (0 < calculatedTs && calculatedTs < Long.MaxValue) { - Right((balance, calculatedTs)) - } else - Left(s"Invalid next block generation time: $calculatedTs") + for { + expectedTS <- pos + .getValidBlockDelay(height, account.publicKey, block.consensusData.baseTarget, balance) + .map(_ + block.timestamp) + .leftMap(_.toString) + result <- Either.cond( + 0 < expectedTS && expectedTS < Long.MaxValue, + (balance, expectedTS), + s"Invalid next block generation time: $expectedTS" + ) + } yield result } else Left(s"Balance $balance of ${account.address} is lower than required for generation") } @@ -284,10 +297,8 @@ class MinerImpl(allChannels: ChannelGroup, val height = blockchainUpdater.height val lastBlock = blockchainUpdater.lastBlock.get for { - _ <- checkAge(height, blockchainUpdater.lastBlockTimestamp.get) - _ <- Either.cond(blockchainUpdater.accountScript(account).isEmpty, - (), - s"Account(${account.toAddress}) is scripted and therefore not allowed to forge blocks") + _ <- checkAge(height, blockchainUpdater.lastBlockTimestamp.get) + _ <- checkScript(account) balanceAndTs <- nextBlockGenerationTime(blockchainSettings.functionalitySettings, height, lastBlock, account) (balance, ts) = balanceAndTs offset = calcOffset(timeService, ts, minerSettings.minimalBlockGenerationOffset) diff --git a/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index d1f0f62face..2add0c7c580 100644 --- a/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -1,7 +1,7 @@ package com.wavesplatform.state.appender import cats.data.EitherT -import com.wavesplatform.consensus.PoSCalculator +import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.metrics._ import com.wavesplatform.mining.Miner import com.wavesplatform.network._ @@ -26,7 +26,7 @@ object BlockAppender extends ScorexLogging with Instrumented { blockchainUpdater: BlockchainUpdater with Blockchain, time: Time, utxStorage: UtxPool, - pos: PoSCalculator, + pos: PoSSelector, settings: WavesSettings, scheduler: Scheduler)(newBlock: Block): Task[Either[ValidationError, Option[BigInt]]] = Task { @@ -49,7 +49,7 @@ object BlockAppender extends ScorexLogging with Instrumented { blockchainUpdater: BlockchainUpdater with Blockchain, time: Time, utxStorage: UtxPool, - pos: PoSCalculator, + pos: PoSSelector, settings: WavesSettings, allChannels: ChannelGroup, peerDatabase: PeerDatabase, diff --git a/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala b/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala index 52309b5f532..bd5ce88e096 100644 --- a/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala +++ b/src/main/scala/com/wavesplatform/state/appender/ExtensionAppender.scala @@ -1,6 +1,6 @@ package com.wavesplatform.state.appender -import com.wavesplatform.consensus.PoSCalculator +import com.wavesplatform.consensus.PoSSelector import com.wavesplatform.metrics.{BlockStats, Instrumented, Metrics} import com.wavesplatform.mining.Miner import com.wavesplatform.network.{InvalidBlockStorage, PeerDatabase, formatBlocks, id} @@ -24,7 +24,7 @@ object ExtensionAppender extends ScorexLogging with Instrumented { def apply(checkpoint: CheckpointService, blockchainUpdater: BlockchainUpdater with Blockchain, utxStorage: UtxPool, - pos: PoSCalculator, + pos: PoSSelector, time: Time, settings: WavesSettings, invalidBlocks: InvalidBlockStorage, diff --git a/src/main/scala/com/wavesplatform/state/appender/package.scala b/src/main/scala/com/wavesplatform/state/appender/package.scala index 3d9b5122017..e724023ac12 100644 --- a/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -1,9 +1,9 @@ package com.wavesplatform.state -import com.wavesplatform.consensus.{GeneratingBalanceProvider, PoSCalculator} +import com.wavesplatform.consensus.{GeneratingBalanceProvider, PoSSelector} import com.wavesplatform.mining._ import com.wavesplatform.network._ -import com.wavesplatform.settings.WavesSettings +import com.wavesplatform.settings.{FunctionalitySettings, WavesSettings} import com.wavesplatform.utx.UtxPool import io.netty.channel.Channel import io.netty.channel.group.ChannelGroup @@ -20,19 +20,14 @@ package object appender extends ScorexLogging { private val MaxTimeDrift: Long = 100 // millis - private val correctBlockId1 = ByteStr.decodeBase58("2GNCYVy7k3kEPXzz12saMtRDeXFKr8cymVsG8Yxx3sZZ75eHj9csfXnGHuuJe7XawbcwjKdifUrV1uMq4ZNCWPf1").get - private val correctBlockId2 = ByteStr.decodeBase58("5uZoDnRKeWZV9Thu2nvJVZ5dBvPB7k2gvpzFD618FMXCbBVBMN2rRyvKBZBhAGnGdgeh2LXEeSr9bJqruJxngsE7").get - private val height1 = 812608 - private val height2 = 813207 - private[appender] def processAndBlacklistOnFailure[A, B]( - ch: Channel, - peerDatabase: PeerDatabase, - miner: Miner, - allChannels: ChannelGroup, - start: => String, - success: => String, - errorPrefix: String)(f: => Task[Either[B, Option[BigInt]]]): Task[Either[B, Option[BigInt]]] = { + ch: Channel, + peerDatabase: PeerDatabase, + miner: Miner, + allChannels: ChannelGroup, + start: => String, + success: => String, + errorPrefix: String)(f: => Task[Either[B, Option[BigInt]]]): Task[Either[B, Option[BigInt]]] = { log.debug(start) f map { @@ -50,7 +45,7 @@ package object appender extends ScorexLogging { private[appender] def appendBlock(checkpoint: CheckpointService, blockchainUpdater: BlockchainUpdater with Blockchain, utxStorage: UtxPool, - pos: PoSCalculator, + pos: PoSSelector, time: Time, settings: WavesSettings)(block: Block): Either[ValidationError, Option[Int]] = for { @@ -68,10 +63,10 @@ package object appender extends ScorexLogging { val balance = GeneratingBalanceProvider.balance(blockchainUpdater, settings.blockchainSettings.functionalitySettings, height, block.sender) Either.cond( GeneratingBalanceProvider.isEffectiveBalanceValid(blockchainUpdater, - settings.blockchainSettings.functionalitySettings, - height, - block, - balance), + settings.blockchainSettings.functionalitySettings, + height, + block, + balance), balance, s"generator's effective balance $balance is less that required for generation" ) @@ -86,68 +81,50 @@ package object appender extends ScorexLogging { maybeDiscardedTxs.map(_ => baseHeight) } - private def blockConsensusValidation(blockchain: Blockchain, settings: WavesSettings, pos: PoSCalculator, currentTs: Long, block: Block)( - genBalance: Int => Either[String, Long]): Either[ValidationError, Unit] = { + private def blockConsensusValidation(blockchain: Blockchain, settings: WavesSettings, pos: PoSSelector, currentTs: Long, block: Block)( + genBalance: Int => Either[String, Long]): Either[ValidationError, Unit] = { - val bcs = settings.blockchainSettings - val fs = bcs.functionalitySettings val blockTime = block.timestamp - val generator = block.signerData.generator - val generatorSignature = - (blockchain.blockAt(blockchain.height - 100) orElse blockchain.lastBlock) - .map(_.consensusData.generationSignature.arr) - .get - val r: Either[ValidationError, Unit] = for { + for { height <- blockchain.heightOf(block.reference).toRight(GenericError(s"height: history does not contain parent ${block.reference}")) - _ <- Either.cond( - height > fs.blockVersion3AfterHeight - || block.version == Block.GenesisBlockVersion - || block.version == Block.PlainBlockVersion, - (), - GenericError(s"Block Version 3 can only appear at height greater than ${fs.blockVersion3AfterHeight}") - ) - _ <- Either.cond(blockTime - currentTs < MaxTimeDrift, (), BlockFromFuture(blockTime)) - _ <- Either.cond( - blockTime < fs.requireSortedTransactionsAfter - || height > fs.dontRequireSortedTransactionsAfter - || block.transactionData.sorted(TransactionsOrdering.InBlock) == block.transactionData, - (), - GenericError("transactions are not sorted") - ) parent <- blockchain.parent(block).toRight(GenericError(s"parent: history does not contain parent ${block.reference}")) - prevBlockData = parent.consensusData - blockData = block.consensusData - ggp = blockchain.parent(parent, 2) - cbt = pos.baseTarget(bcs.genesisSettings.averageBlockDelay.toSeconds, - height, - parent.consensusData.baseTarget, - parent.timestamp, - ggp.map(_.timestamp), - blockTime) - bbt = blockData.baseTarget - _ <- Either.cond(cbt == bbt, (), GenericError(s"declared baseTarget $bbt does not match calculated baseTarget $cbt")) - calcGs = pos.generatorSignature(generatorSignature, generator.publicKey) - blockGs = blockData.generationSignature.arr - _ <- Either.cond( - calcGs.sameElements(blockGs), - (), - GenericError( - s"declared generation signature ${blockData.generationSignature.base58} does not match calculated generation signature ${ByteStr(calcGs).base58}") - ) + grandParent = blockchain.parent(parent, 2) effectiveBalance <- genBalance(height).left.map(GenericError(_)) - minValidBlockTime = parent.timestamp + pos.validBlockDelay(blockGs, parent.consensusData.baseTarget, effectiveBalance) + _ <- validateBlockVersion(height, block, settings.blockchainSettings.functionalitySettings) + _ <- Either.cond(blockTime - currentTs < MaxTimeDrift, (), BlockFromFuture(blockTime)) + _ <- validateTransactionSorting(height, block, settings.blockchainSettings.functionalitySettings) + _ <- pos.validateBaseTarget(height, block, parent, grandParent) + _ <- pos.validateGeneratorSignature(height, block) + _ <- pos.validateBlockDelay(height, block, parent, effectiveBalance) + } yield () + }.left.map { + case GenericError(x) => GenericError(s"Block $block is invalid: $x") + case x => x + } + + private def validateBlockVersion(height: Int, block: Block, fs: FunctionalitySettings): Either[ValidationError, Unit] = { + val version3Height = fs.blockVersion3AfterHeight + Either.cond( + height > version3Height + || block.version == Block.GenesisBlockVersion + || block.version == Block.PlainBlockVersion, + (), + GenericError(s"Block Version 3 can only appear at height greater than $version3Height") + ) + } + + private def validateTransactionSorting(height: Int, block: Block, settings: FunctionalitySettings): Either[ValidationError, Unit] = { + val blockTime = block.timestamp + for { _ <- Either.cond( - blockTime > minValidBlockTime - || (height == height1 && block.uniqueId == correctBlockId1) || (height == height2 && block.uniqueId == correctBlockId2), + blockTime < settings.requireSortedTransactionsAfter + || height > settings.dontRequireSortedTransactionsAfter + || block.transactionData.sorted(TransactionsOrdering.InBlock) == block.transactionData, (), - GenericError(s"calculated time $minValidBlockTime < block time $blockTime") + GenericError("transactions are not sorted") ) } yield () - - r.left.map { - case GenericError(x) => GenericError(s"Block $block is invalid: $x") - case x => x - } } + } diff --git a/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala b/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala index b525a637b18..8dd6bf8c8a8 100755 --- a/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala +++ b/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala @@ -2,6 +2,7 @@ package scorex.consensus.nxt.api.http import akka.http.scaladsl.server.Route import com.wavesplatform.consensus.GeneratingBalanceProvider +import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.settings.{FunctionalitySettings, RestAPISettings} import com.wavesplatform.state.Blockchain import io.swagger.annotations._ @@ -78,6 +79,11 @@ case class NxtConsensusApiRoute(settings: RestAPISettings, blockchain: Blockchai @Path("/algo") @ApiOperation(value = "Consensus algo", notes = "Shows which consensus algo being using", httpMethod = "GET") def algo: Route = (path("algo") & get) { - complete(Json.obj("consensusAlgo" -> "proof-of-stake (PoS)")) + complete( + if (blockchain.activatedFeatures.contains(BlockchainFeatures.FairPoS.id)) + Json.obj("consensusAlgo" -> "Fair Proof-of-Stake (FairPoS)") + else + Json.obj("consensusAlgo" -> "proof-of-stake (PoS)") + ) } } From 19c884645e08c7873876b8c3e7ff483bf484cc30 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 21 May 2018 13:34:22 +0300 Subject: [PATCH 08/52] NODE-725 Proof with empty string cannot be decoded --- .../it/sync/LeaseSmartContractsTestSuite.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/it/src/test/scala/com/wavesplatform/it/sync/LeaseSmartContractsTestSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/LeaseSmartContractsTestSuite.scala index 0697504cf8b..a77aa3de231 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/LeaseSmartContractsTestSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/LeaseSmartContractsTestSuite.scala @@ -77,9 +77,8 @@ class LeaseSmartContractsTestSuite extends BaseTransactionSuite with CancelAfter val sigLeasingA = ByteStr(crypto.sign(acc0, unsignedLeasing.bodyBytes())) val sigLeasingC = ByteStr(crypto.sign(acc2, unsignedLeasing.bodyBytes())) - /* issue https://wavesplatform.atlassian.net/browse/NODE-725 */ val signedLeasing = - unsignedLeasing.copy(proofs = Proofs(Seq(sigLeasingA, ByteStr("0".getBytes()), sigLeasingC))) + unsignedLeasing.copy(proofs = Proofs(Seq(sigLeasingA, ByteStr.empty, sigLeasingC))) val leasingId = sender.signedBroadcast(signedLeasing.json() + ("type" -> JsNumber(LeaseTransactionV2.typeId.toInt))).id @@ -105,9 +104,8 @@ class LeaseSmartContractsTestSuite extends BaseTransactionSuite with CancelAfter val sigLeasingCancelA = ByteStr(crypto.sign(acc0, unsignedCancelLeasing.bodyBytes())) val sigLeasingCancelB = ByteStr(crypto.sign(acc1, unsignedCancelLeasing.bodyBytes())) - /* issue https://wavesplatform.atlassian.net/browse/NODE-725 */ val signedLeasingCancel = - unsignedCancelLeasing.copy(proofs = Proofs(Seq(ByteStr("0".getBytes()), sigLeasingCancelA, sigLeasingCancelB))) + unsignedCancelLeasing.copy(proofs = Proofs(Seq(ByteStr.empty, sigLeasingCancelA, sigLeasingCancelB))) val leasingCancelId = sender.signedBroadcast(signedLeasingCancel.json() + ("type" -> JsNumber(LeaseCancelTransactionV2.typeId.toInt))).id From e10596474b6fdaa3a4e51e1c6cf06070989496b3 Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Mon, 21 May 2018 15:17:39 +0300 Subject: [PATCH 09/52] NODE-732 Add some binary operations. --- .../v1/evaluator/ctx/impl/PureContext.scala | 7 +- .../lang/v1/parser/BinaryOperation.scala | 71 +++++++++++++++---- .../wavesplatform/lang/v1/parser/Parser.scala | 8 +-- .../lang/v1/testing/ScriptGen.scala | 59 +++++++++++++-- 4 files changed, 119 insertions(+), 26 deletions(-) diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala index b04bd583573..ee42cce14c1 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala @@ -69,7 +69,7 @@ object PureContext { val uMinus = PredefFunction("-", 1, LONG, List("n" -> LONG)) { case (n: Long) :: Nil => { - Right(-n) + Right(Math.negateExact(n)) } case _ => ??? } @@ -94,6 +94,7 @@ object PureContext { } val sumLong = createTryOp(SUM_OP, LONG, LONG)(Math.addExact) + val subLong = createTryOp(SUB_OP, LONG, LONG)(Math.subtractExact) val sumString = createOp(SUM_OP, STRING, STRING)(_ + _) val sumByteVector = createOp(SUM_OP, BYTEVECTOR, BYTEVECTOR)((a, b) => ByteVector(a.toArray ++ b.toArray)) val eqLong = createOp(EQ_OP, LONG, BOOLEAN)(_ == _) @@ -102,8 +103,10 @@ object PureContext { val eqString = createOp(EQ_OP, STRING, BOOLEAN)(_ == _) val ge = createOp(GE_OP, LONG, BOOLEAN)(_ >= _) val gt = createOp(GT_OP, LONG, BOOLEAN)(_ > _) + val bge = createOp(GE_OP, BOOLEAN, BOOLEAN)(_ >= _) + val bgt = createOp(GT_OP, BOOLEAN, BOOLEAN)(_ > _) - val operators: Seq[PredefFunction] = Seq(sumLong, sumString, sumByteVector, eqLong, eqByteVector, eqBool, eqString, ge, gt, getElement, getListSize, uMinus, uNot) + val operators: Seq[PredefFunction] = Seq(sumLong, sumString, sumByteVector, eqLong, eqByteVector, eqBool, eqString, ge, gt, bge, bgt, getElement, getListSize, uMinus, uNot) lazy val instance = EvaluationContext.build(types = Seq.empty, diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala index fad62094165..cd657b3928c 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala @@ -1,25 +1,66 @@ package com.wavesplatform.lang.v1.parser -sealed trait BinaryOperation +import Expressions._ +import fastparse.all._ + +sealed abstract class BinaryOperation { + val func: String + val parser: P[Any] = P(func) + def expr(op1: EXPR)(op2: EXPR): EXPR = { + BINARY_OP(op1, this, op2) + } +} object BinaryOperation { - val opsByPriority = List[(String, BinaryOperation)]( - "||" -> OR_OP, - "&&" -> AND_OP, - "==" -> EQ_OP, - ">=" -> GE_OP, - ">" -> GT_OP, - "+" -> SUM_OP + val opsByPriority = List[BinaryOperation]( + OR_OP, + AND_OP, + EQ_OP, + GE_OP, + GT_OP, + LE_OP, + LT_OP, + SUM_OP, + SUB_OP ) - val opsToFunctions = opsByPriority.map { case (str, op) => op -> str }.toMap + def opsToFunctions(op: BinaryOperation) = op.func - case object SUM_OP extends BinaryOperation - case object AND_OP extends BinaryOperation - case object OR_OP extends BinaryOperation - case object EQ_OP extends BinaryOperation - case object GT_OP extends BinaryOperation - case object GE_OP extends BinaryOperation + case object OR_OP extends BinaryOperation { + val func = "||" + } + case object AND_OP extends BinaryOperation { + val func = "&&" + } + case object EQ_OP extends BinaryOperation { + val func = "==" + } + case object GE_OP extends BinaryOperation { + val func = ">=" + } + case object GT_OP extends BinaryOperation { + val func = ">" + } + case object SUM_OP extends BinaryOperation { + val func = "+" + } + case object SUB_OP extends BinaryOperation { + val func = "-" + } + case object LE_OP extends BinaryOperation { + val func = ">=" + override val parser = P("<=") + override def expr(op1: EXPR)(op2: EXPR): EXPR = { + BINARY_OP(op2, LE_OP, op1) + } + } + case object LT_OP extends BinaryOperation { + val func = ">" + override val parser = P("<") + override def expr(op1: EXPR)(op2: EXPR): EXPR = { + BINARY_OP(op2, LT_OP, op1) + } + } } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala index ed67621811e..175024abd5b 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala @@ -93,13 +93,13 @@ object Parser { } } - private def binaryOp(rest: List[(String, BinaryOperation)]): P[EXPR] = rest match { + private def binaryOp(rest: List[(BinaryOperation)]): P[EXPR] = rest match { case Nil => unaryOp(unaryOps) - case (lessPriorityOp, kind) :: restOps => + case kind :: restOps => val operand = binaryOp(restOps) - P(operand ~ (lessPriorityOp.!.map(_ => kind) ~ operand).rep()).map { + P(operand ~ (kind.parser.!.map(_ => kind) ~ operand).rep()).map { case (left: EXPR, r: Seq[(BinaryOperation, EXPR)]) => - r.foldLeft(left) { case (acc, (currKind, currOperand)) => BINARY_OP(acc, currKind, currOperand) } + r.foldLeft(left) { case (acc, (currKind, currOperand)) => currKind.expr(acc)(currOperand) } } } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index 8975db1a816..bf19b3af1f8 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -10,7 +10,7 @@ trait ScriptGen { def CONST_LONGgen: Gen[(EXPR, Long)] = Gen.choose(Long.MinValue, Long.MaxValue).map(v => (CONST_LONG(v), v)) def BOOLgen(gas: Int): Gen[(EXPR,Boolean)] = - if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1)) + if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), BGEgen(gas - 1), BGTgen(gas - 1), EQ_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1)) else Gen.const((TRUE, true)) def SUMgen(gas: Int): Gen[(EXPR, Long)] = @@ -20,19 +20,58 @@ trait ScriptGen { if((BigInt(v1) + BigInt(v2)).isValidLong) } yield (BINARY_OP(i1, SUM_OP, i2), (v1 + v2)) - def INTGen(gas: Int): Gen[(EXPR, Long)] = if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), IF_INTgen(gas - 1), INTGen(gas-1).map(e => (FUNCTION_CALL("-",List(e._1)), -e._2))) else CONST_LONGgen + def SUBgen(gas: Int): Gen[(EXPR, Long)] = + for { + (i1, v1) <- INTGen((gas - 2) / 2) + (i2, v2) <- INTGen((gas - 2) / 2) + if((BigInt(v1) - BigInt(v2)).isValidLong) + } yield (BINARY_OP(i1, SUB_OP, i2), (v1 + v2)) + + def INTGen(gas: Int): Gen[(EXPR, Long)] = if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), IF_INTgen(gas - 1), INTGen(gas-1).filter(v => (-BigInt(v._2)).isValidLong).map(e => (FUNCTION_CALL("-",List(e._1)), -e._2))) else CONST_LONGgen + + def BGEgen(gas: Int): Gen[(EXPR, Boolean)] = + for { + dir <- Gen.oneOf(true, false) + (i1, v1) <- BOOLgen((gas - 2) / 2) + (i2, v2) <- BOOLgen((gas - 2) / 2) + } yield if(dir) { + (BINARY_OP(i1, GE_OP, i2), (v1 >= v2)) + } else { + (BINARY_OP(i2, LE_OP, i1), (v1 <= v2)) + } + + def BGTgen(gas: Int): Gen[(EXPR, Boolean)] = + for { + dir <- Gen.oneOf(true, false) + (i1, v1) <- BOOLgen((gas - 2) / 2) + (i2, v2) <- BOOLgen((gas - 2) / 2) + } yield if(dir) { + (BINARY_OP(i1, GE_OP, i2), (v1 > v2)) + } else { + (BINARY_OP(i2, LE_OP, i1), (v1 < v2)) + } def GEgen(gas: Int): Gen[(EXPR, Boolean)] = for { + dir <- Gen.oneOf(true, false) (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - } yield (BINARY_OP(i1, GE_OP, i2), (v1 >= v2)) + } yield if(dir) { + (BINARY_OP(i1, GE_OP, i2), (v1 >= v2)) + } else { + (BINARY_OP(i2, LE_OP, i1), (v1 <= v2)) + } def GTgen(gas: Int): Gen[(EXPR, Boolean)] = for { + dir <- Gen.oneOf(true, false) (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - } yield (BINARY_OP(i1, GT_OP, i2), (v1 > v2)) + } yield if(dir) { + (BINARY_OP(i1, GT_OP, i2), (v1 > v2)) + } else { + (BINARY_OP(i2, LT_OP, i1), (v1 < v2)) + } def EQ_INTgen(gas: Int): Gen[(EXPR, Boolean)] = for { @@ -96,7 +135,7 @@ trait ScriptGen { for { pred <- whitespaces post <- whitespaces - } yield pred + expr + post + } yield s" $expr " //pred + expr + post def toString(expr: EXPR): Gen[String] = expr match { case CONST_LONG(x) => withWhitespaces(s"$x") @@ -106,6 +145,16 @@ trait ScriptGen { case FALSE => withWhitespaces("false") case FUNCTION_CALL("-", List(CONST_LONG(v))) if (v>=0) => s"-($v)" case FUNCTION_CALL(op, List(e)) => toString(e).map(e => s"$op$e") + case BINARY_OP(x, LE_OP, y) => + for { + arg2 <- toString(x) + arg1 <- toString(y) + } yield s"($arg1<=$arg2)" + case BINARY_OP(x, LT_OP, y) => + for { + arg2 <- toString(x) + arg1 <- toString(y) + } yield s"($arg1<$arg2)" case BINARY_OP(x, op: BinaryOperation, y) => for { arg1 <- toString(x) From c10ffc7128c5ebf6718d38734e5987a08cecc74d Mon Sep 17 00:00:00 2001 From: "Nastya.Urlapova" Date: Mon, 21 May 2018 15:25:27 +0300 Subject: [PATCH 10/52] NODE-717 Supported extended fields for MassTransfer in SmartContracts testing --- .../sync/MassTransferSmartContractSuite.scala | 123 ++++++++++++++++++ .../it/sync/SetScriptTransactionSuite.scala | 55 -------- 2 files changed, 123 insertions(+), 55 deletions(-) create mode 100644 it/src/test/scala/com/wavesplatform/it/sync/MassTransferSmartContractSuite.scala diff --git a/it/src/test/scala/com/wavesplatform/it/sync/MassTransferSmartContractSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/MassTransferSmartContractSuite.scala new file mode 100644 index 00000000000..21fa4a6010a --- /dev/null +++ b/it/src/test/scala/com/wavesplatform/it/sync/MassTransferSmartContractSuite.scala @@ -0,0 +1,123 @@ +package com.wavesplatform.it.sync + +import com.wavesplatform.crypto +import com.wavesplatform.it.api.SyncHttpApi._ +import com.wavesplatform.it.transactions.BaseTransactionSuite +import com.wavesplatform.it.util._ +import com.wavesplatform.lang.v1.compiler.CompilerV1 +import com.wavesplatform.lang.v1.parser.Parser +import com.wavesplatform.state._ +import com.wavesplatform.utils.dummyTypeCheckerContext +import org.scalatest.CancelAfterFailure +import play.api.libs.json.JsNumber +import scorex.crypto.encode.Base58 +import scorex.transaction.Proofs +import scorex.transaction.smart.SetScriptTransaction +import scorex.transaction.smart.script.v1.ScriptV1 +import scorex.transaction.transfer.MassTransferTransaction.Transfer +import scorex.transaction.transfer._ +import scala.concurrent.duration._ + +/* +Scenario: +every month a foundation makes payments from two MassTransactions(type == 11): +1) 80% to users +2) 10% as tax and 10% to bank go after 30sec of payment from step 1) + */ + +class MassTransferSmartContractSuite extends BaseTransactionSuite with CancelAfterFailure { + private val fourthAddress: String = sender.createAddress() + private val transferAmount: Long = 1.waves + private val fee: Long = 0.001.waves + private val massTransferFee = 0.004.waves + 0.0005.waves * 4 + + test("airdrop emulation via MassTransfer") { + val scriptText = { + val untyped = Parser(s""" + let commonAmount = (tx.transfers[0].amount + tx.transfers[1].amount) + let totalAmountToUsers = commonAmount == 800000000 + let totalAmountToGov = commonAmount == 200000000 + let massTransferType = ((tx.type == 11) && (size(tx.transfers) == 2)) + + let accountPK = base58'${ByteStr(sender.publicKey.publicKey)}' + let accSig = sigVerify(tx.bodyBytes,tx.proofs[0],accountPK) + + let txToUsers = (massTransferType && totalAmountToUsers) + + let massTransferTx = getTransactionById(tx.proofs[1]) + let massTransferTime = if(isDefined(massTransferTx)) then Some(extract(massTransferTx).timestamp) else None + + let txToGov = (massTransferType && totalAmountToGov) + let txToGovComplete = if(isDefined(massTransferTime)) then (tx.timestamp > (extract(massTransferTime) + 30000)) else false + + (txToGovComplete && accSig && txToGov) || (txToUsers && accSig) + """.stripMargin).get.value + CompilerV1(dummyTypeCheckerContext, untyped).explicitGet() + } + + // set script + val script = ScriptV1(scriptText).explicitGet() + val setScriptTransaction = SetScriptTransaction + .selfSigned(SetScriptTransaction.supportedVersions.head, sender.privateKey, Some(script), fee, System.currentTimeMillis()) + .explicitGet() + + val setScriptId = sender + .signedBroadcast(setScriptTransaction.json() + ("type" -> JsNumber(SetScriptTransaction.typeId.toInt))) + .id + + nodes.waitForHeightAriseAndTxPresent(setScriptId) + + sender.addressScriptInfo(sender.address).scriptText.isEmpty shouldBe false + + //save time + val currTime = System.currentTimeMillis() + + //make transfer to users + val transfers = + MassTransferTransaction + .parseTransfersList(List(Transfer(thirdAddress, 4 * transferAmount), Transfer(secondAddress, 4 * transferAmount))) + .right + .get + + val unsigned = + MassTransferTransaction + .create(1, None, sender.publicKey, transfers, currTime, massTransferFee, Array.emptyByteArray, Proofs.empty) + .explicitGet() + + val accountSig = ByteStr(crypto.sign(sender.privateKey, unsigned.bodyBytes())) + val signed = unsigned.copy(proofs = Proofs(Seq(accountSig))) + val toUsersID = sender.signedBroadcast(signed.json() + ("type" -> JsNumber(MassTransferTransaction.typeId.toInt))).id + + nodes.waitForHeightAriseAndTxPresent(toUsersID) + + //make transfer with incorrect time + val heightBefore = sender.height + + val transfersToGov = + MassTransferTransaction.parseTransfersList(List(Transfer(firstAddress, transferAmount), Transfer(fourthAddress, transferAmount))).right.get + + val unsignedToGov = + MassTransferTransaction + .create(1, None, sender.publicKey, transfersToGov, currTime, massTransferFee, Array.emptyByteArray, Proofs.empty) + .explicitGet() + val accountSigToGovFail = ByteStr(crypto.sign(sender.privateKey, unsignedToGov.bodyBytes())) + val signedToGovFail = unsignedToGov.copy(proofs = Proofs(Seq(accountSigToGovFail))) + + assertBadRequestAndResponse(sender.signedBroadcast(signedToGovFail.json() + ("type" -> JsNumber(MassTransferTransaction.typeId.toInt))), + "Reason: TransactionNotAllowedByScript") + + //make correct transfer to government after some time + sender.waitForHeight(heightBefore + 5, 1.minutes) + + val unsignedToGovSecond = + MassTransferTransaction + .create(1, None, sender.publicKey, transfersToGov, System.currentTimeMillis(), massTransferFee, Array.emptyByteArray, Proofs.empty) + .explicitGet() + + val accountSigToGov = ByteStr(crypto.sign(sender.privateKey, unsignedToGovSecond.bodyBytes())) + val signedToGovGood = unsignedToGovSecond.copy(proofs = Proofs(Seq(accountSigToGov, ByteStr(Base58.decode(toUsersID).get)))) + val massTransferID = sender.signedBroadcast(signedToGovGood.json() + ("type" -> JsNumber(MassTransferTransaction.typeId.toInt))).id + + nodes.waitForHeightAriseAndTxPresent(massTransferID) + } +} diff --git a/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala index 3128d70c1e6..ebd65428b3d 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala @@ -15,9 +15,6 @@ import scorex.transaction.Proofs import scorex.transaction.transfer._ import scorex.transaction.smart.SetScriptTransaction import scorex.transaction.smart.script.v1.ScriptV1 -import scorex.transaction.transfer.MassTransferTransaction.Transfer - -import scala.concurrent.duration._ class SetScriptTransactionSuite extends BaseTransactionSuite with CancelAfterFailure { private def pkFromAddress(address: String) = PrivateKeyAccount.fromSeed(sender.seed(address)).right.get @@ -172,56 +169,4 @@ class SetScriptTransactionSuite extends BaseTransactionSuite with CancelAfterFai val txId = sender.signedBroadcast(tx.json() + ("type" -> JsNumber(TransferTransactionV2.typeId.toInt))).id nodes.waitForHeightAriseAndTxPresent(txId) } - - test("make masstransfer after some height") { - val heightBefore = sender.height - - val scriptText = { - val untyped = Parser(s""" - let A = base58'${ByteStr(acc3.publicKey)}' - - let AC = sigVerify(tx.bodyBytes,tx.proofs[0],A) - let heightVerification = if (height > $heightBefore + 10) then true else false - - AC && heightVerification - """.stripMargin).get.value - CompilerV1(dummyTypeCheckerContext, untyped).explicitGet() - } - - val script = ScriptV1(scriptText).explicitGet() - val setScriptTransaction = SetScriptTransaction - .selfSigned(SetScriptTransaction.supportedVersions.head, acc0, Some(script), fee, System.currentTimeMillis()) - .explicitGet() - - val setScriptId = sender - .signedBroadcast(setScriptTransaction.json() + ("type" -> JsNumber(SetScriptTransaction.typeId.toInt))) - .id - - nodes.waitForHeightAriseAndTxPresent(setScriptId) - - sender.addressScriptInfo(firstAddress).scriptText.isEmpty shouldBe false - - val transfers = - MassTransferTransaction.parseTransfersList(List(Transfer(thirdAddress, transferAmount), Transfer(secondAddress, transferAmount))).right.get - - val massTransferFee = 0.004.waves + 0.0005.waves * 4 - - val unsigned = - MassTransferTransaction - .create(1, None, acc0, transfers, System.currentTimeMillis(), massTransferFee, Array.emptyByteArray, Proofs.empty) - .explicitGet() - - val notarySig = ByteStr(crypto.sign(acc3, unsigned.bodyBytes())) - - val signed = unsigned.copy(proofs = Proofs(Seq(notarySig))) - - assertBadRequestAndResponse(sender.signedBroadcast(signed.json() + ("type" -> JsNumber(MassTransferTransaction.typeId.toInt))), - "Reason: TransactionNotAllowedByScript") - - sender.waitForHeight(heightBefore + 11, 2.minutes) - - val massTransferID = sender.signedBroadcast(signed.json() + ("type" -> JsNumber(MassTransferTransaction.typeId.toInt))).id - - nodes.waitForHeightAriseAndTxPresent(massTransferID) - } } From 864cf7a24afa8be26319451c524282ed2eda90b5 Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Mon, 21 May 2018 17:10:19 +0300 Subject: [PATCH 11/52] NODE-732 Fix binary operator '-'. --- .../lang/v1/evaluator/ctx/impl/PureContext.scala | 2 +- .../com/wavesplatform/lang/v1/testing/ScriptGen.scala | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala index ee42cce14c1..8c90c5ab586 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala @@ -106,7 +106,7 @@ object PureContext { val bge = createOp(GE_OP, BOOLEAN, BOOLEAN)(_ >= _) val bgt = createOp(GT_OP, BOOLEAN, BOOLEAN)(_ > _) - val operators: Seq[PredefFunction] = Seq(sumLong, sumString, sumByteVector, eqLong, eqByteVector, eqBool, eqString, ge, gt, bge, bgt, getElement, getListSize, uMinus, uNot) + val operators: Seq[PredefFunction] = Seq(sumLong, subLong, sumString, sumByteVector, eqLong, eqByteVector, eqBool, eqString, ge, gt, bge, bgt, getElement, getListSize, uMinus, uNot) lazy val instance = EvaluationContext.build(types = Seq.empty, diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index bf19b3af1f8..67741a88cd5 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -25,9 +25,9 @@ trait ScriptGen { (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) if((BigInt(v1) - BigInt(v2)).isValidLong) - } yield (BINARY_OP(i1, SUB_OP, i2), (v1 + v2)) + } yield (BINARY_OP(i1, SUB_OP, i2), (v1 - v2)) - def INTGen(gas: Int): Gen[(EXPR, Long)] = if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), IF_INTgen(gas - 1), INTGen(gas-1).filter(v => (-BigInt(v._2)).isValidLong).map(e => (FUNCTION_CALL("-",List(e._1)), -e._2))) else CONST_LONGgen + def INTGen(gas: Int): Gen[(EXPR, Long)] = if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), SUBgen(gas - 1), IF_INTgen(gas - 1), INTGen(gas-1).filter(v => (-BigInt(v._2)).isValidLong).map(e => (FUNCTION_CALL("-",List(e._1)), -e._2))) else CONST_LONGgen def BGEgen(gas: Int): Gen[(EXPR, Boolean)] = for { @@ -46,9 +46,9 @@ trait ScriptGen { (i1, v1) <- BOOLgen((gas - 2) / 2) (i2, v2) <- BOOLgen((gas - 2) / 2) } yield if(dir) { - (BINARY_OP(i1, GE_OP, i2), (v1 > v2)) + (BINARY_OP(i1, GT_OP, i2), (v1 > v2)) } else { - (BINARY_OP(i2, LE_OP, i1), (v1 < v2)) + (BINARY_OP(i2, LT_OP, i1), (v1 < v2)) } def GEgen(gas: Int): Gen[(EXPR, Boolean)] = From 3c458ace0af7a9a506c7b09835c3a94c7d66b646 Mon Sep 17 00:00:00 2001 From: "Nastya.Urlapova" Date: Thu, 17 May 2018 19:34:22 +0300 Subject: [PATCH 12/52] NODE-755 Back-compatibility unit tests for transactions --- .../BurnTransactionSpecification.scala | 71 ++++++++++++++++++- .../CreateAliasTransactionSpecification.scala | 64 +++++++++++++++++ .../DataTransactionSpecification.scala | 52 ++++++++++++++ .../IssueTransactionV1Specification.scala | 43 ++++++++++- .../IssueTransactionV2Specification.scala | 48 ++++++++++++- .../LeaseCancelTransactionSpecification.scala | 71 ++++++++++++++++++- .../LeaseTransactionSpecification.scala | 69 +++++++++++++++++- ...MassTransferTransactionSpecification.scala | 57 ++++++++++++++- .../ReissueTransactionV1Specification.scala | 39 +++++++++- .../ReissueTransactionV2Specification.scala | 51 +++++++++++++ .../TransferTransactionV1Specification.scala | 39 ++++++++++ .../TransferTransactionV2Specification.scala | 41 +++++++++++ 12 files changed, 638 insertions(+), 7 deletions(-) create mode 100644 src/test/scala/scorex/transaction/ReissueTransactionV2Specification.scala diff --git a/src/test/scala/scorex/transaction/BurnTransactionSpecification.scala b/src/test/scala/scorex/transaction/BurnTransactionSpecification.scala index a6b51200127..189875ae118 100644 --- a/src/test/scala/scorex/transaction/BurnTransactionSpecification.scala +++ b/src/test/scala/scorex/transaction/BurnTransactionSpecification.scala @@ -3,7 +3,11 @@ package scorex.transaction import com.wavesplatform.TransactionGen import org.scalatest._ import org.scalatest.prop.PropertyChecks -import scorex.transaction.assets.BurnTransaction +import play.api.libs.json.Json +import scorex.transaction.assets.{BurnTransaction, BurnTransactionV1, BurnTransactionV2} +import scorex.account.PublicKeyAccount +import com.wavesplatform.state.ByteStr +import scorex.crypto.encode.Base58 class BurnTransactionSpecification extends PropSpec with PropertyChecks with Matchers with TransactionGen { @@ -21,4 +25,69 @@ class BurnTransactionSpecification extends PropSpec with PropertyChecks with Mat } } + property("JSON format validation for BurnTransactionV1") { + val js = Json.parse("""{ + "type": 6, + "id": "Ci1q7y7Qq2C2GDH7YVXsQ8w5vRRKYeoYTp9J76AXw8TZ", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000000, + "timestamp": 1526287561757, + "signature": "uapJcAJQryBhWThU43rYgMNmvdT7kY747vx5BBgxr2KvaeTRx8Vsuh4yu1JxBymU9LnAoo1zjQcPrWSuhi6dVPE", + "chainId": null, + "version": 1, + "assetId": "9ekQuYn92natMnMq8KqeGK3Nn7cpKd3BvPEGgD6fFyyz", + "amount": 10000000000 + } + """) //TODO: change to "quantity" after NODE-765 fix + + val tx = BurnTransactionV1 + .create( + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + ByteStr(Base58.decode("9ekQuYn92natMnMq8KqeGK3Nn7cpKd3BvPEGgD6fFyyz").get), + 10000000000L, + 100000000L, + 1526287561757L, + ByteStr(Base58.decode("uapJcAJQryBhWThU43rYgMNmvdT7kY747vx5BBgxr2KvaeTRx8Vsuh4yu1JxBymU9LnAoo1zjQcPrWSuhi6dVPE").get) + ) + .right + .get + js shouldEqual tx.json() + } + + property("JSON format validation for BurnTransactionV2") { + val js = Json.parse("""{ + "type": 6, + "id": "6QA1sLV53euVCX5fFemNuEyRVdQ5JYo5dWDsCmtKADRc", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000000, + "timestamp": 1526287561757, + "proofs": [ + "3NcEv6tcVMuXkTJwiqW4J3GMCTe8iSLY7neEfNZonp59eTQEZXYPQWs565CRUctDrvcbtmsRgWvnN7BnFZ1AVZ1H" + ], + "chainId": 84, + "version": 2, + "assetId": "9ekQuYn92natMnMq8KqeGK3Nn7cpKd3BvPEGgD6fFyyz", + "amount": 10000000000 + } + """) //TODO: change to "quantity" after NODE-765 fix + + val tx = BurnTransactionV2 + .create( + 2, + 'T', + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + ByteStr(Base58.decode("9ekQuYn92natMnMq8KqeGK3Nn7cpKd3BvPEGgD6fFyyz").get), + 10000000000L, + 100000000L, + 1526287561757L, + Proofs(Seq(ByteStr(Base58.decode("3NcEv6tcVMuXkTJwiqW4J3GMCTe8iSLY7neEfNZonp59eTQEZXYPQWs565CRUctDrvcbtmsRgWvnN7BnFZ1AVZ1H").get))) + ) + .right + .get + + js shouldEqual tx.json() + } + } diff --git a/src/test/scala/scorex/transaction/CreateAliasTransactionSpecification.scala b/src/test/scala/scorex/transaction/CreateAliasTransactionSpecification.scala index 6272404c343..cf31d8cd9da 100644 --- a/src/test/scala/scorex/transaction/CreateAliasTransactionSpecification.scala +++ b/src/test/scala/scorex/transaction/CreateAliasTransactionSpecification.scala @@ -1,9 +1,13 @@ package scorex.transaction import com.wavesplatform.TransactionGen +import com.wavesplatform.state.ByteStr import org.scalatest._ import org.scalatest.prop.PropertyChecks +import play.api.libs.json.Json import scorex.account.{Alias, PrivateKeyAccount} +import scorex.account.PublicKeyAccount +import scorex.crypto.encode.Base58 class CreateAliasTransactionSpecification extends PropSpec with PropertyChecks with Matchers with TransactionGen { @@ -29,4 +33,64 @@ class CreateAliasTransactionSpecification extends PropSpec with PropertyChecks w tx1.id() shouldBe tx2.id() } } + + property("JSON format validation for CreateAliasTransactionV1") { + val js = Json.parse("""{ + "type": 10, + "id": "7acjQQWJAharrgzb4Z6jo3eeAKAGPmLkHTPtvBTKaiug", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000, + "timestamp": 1526910778245, + "signature": "CC1jQ4qkuVfMvB2Kpg2Go6QKXJxUFC8UUswUxBsxwisrR8N5s3Yc8zA6dhjTwfWKfdouSTAnRXCxTXb3T6pJq3T", + "version": 1, + "alias": "myalias" + } + """) + + val tx = CreateAliasTransactionV1 + .create( + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + Alias.buildWithCurrentNetworkByte("myalias").right.get, + 100000, + 1526910778245L, + ByteStr(Base58.decode("CC1jQ4qkuVfMvB2Kpg2Go6QKXJxUFC8UUswUxBsxwisrR8N5s3Yc8zA6dhjTwfWKfdouSTAnRXCxTXb3T6pJq3T").get) + ) + .right + .get + + js shouldEqual tx.json() + } + + property("JSON format validation for CreateAliasTransactionV2") { + val js = Json.parse("""{ + "type": 10, + "id": "7acjQQWJAharrgzb4Z6jo3eeAKAGPmLkHTPtvBTKaiug", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000, + "timestamp": 1526910778245, + "proofs": [ + "26U7rQTwpdma5GYSZb5bNygVCtSuWL6DKet1Nauf5J57v19mmfnq434YrkKYJqvYt2ydQBUT3P7Xgj5ZVDVAcc5k" + ], + "version": 2, + "alias": "myalias" + } + """) + + val tx = CreateAliasTransactionV2 + .create( + 2, + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + Alias.buildWithCurrentNetworkByte("myalias").right.get, + 100000, + 1526910778245L, + Proofs(Seq(ByteStr(Base58.decode("26U7rQTwpdma5GYSZb5bNygVCtSuWL6DKet1Nauf5J57v19mmfnq434YrkKYJqvYt2ydQBUT3P7Xgj5ZVDVAcc5k").get))) + ) + .right + .get + + js shouldEqual tx.json() + } + } diff --git a/src/test/scala/scorex/transaction/DataTransactionSpecification.scala b/src/test/scala/scorex/transaction/DataTransactionSpecification.scala index 94de8f3dc14..d32f09ffba8 100644 --- a/src/test/scala/scorex/transaction/DataTransactionSpecification.scala +++ b/src/test/scala/scorex/transaction/DataTransactionSpecification.scala @@ -8,6 +8,7 @@ import org.scalacheck.{Arbitrary, Gen} import org.scalatest._ import org.scalatest.prop.PropertyChecks import play.api.libs.json.{Format, Json} +import scorex.account.{Alias, PublicKeyAccount} import scorex.api.http.SignedDataRequest import scorex.crypto.encode.Base58 import scorex.transaction.DataTransaction.MaxEntryCount @@ -130,4 +131,55 @@ class DataTransactionSpecification extends PropSpec with PropertyChecks with Mat negativeFeeEi shouldBe Left(ValidationError.InsufficientFee()) } } + + property(testName = "JSON format validation") { + val js = Json.parse("""{ + "type": 12, + "id": "87SfuGJXH1cki2RGDH7WMTGnTXeunkc5mEjNKmmMdRzM", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000, + "timestamp": 1526911531530, + "proofs": [ + "32mNYSefBTrkVngG5REkmmGAVv69ZvNhpbegmnqDReMTmXNyYqbECPgHgXrX2UwyKGLFS45j7xDFyPXjF8jcfw94" + ], + "version": 1, + "data": [ + { + "key": "int", + "type": "integer", + "value": 24 + }, + { + "key": "bool", + "type": "boolean", + "value": true + }, + { + "key": "blob", + "type": "binary", + "value": "BzWHaQU" + } + ] + } + """) + + val entry1 = LongDataEntry("int", 24) + val entry2 = BooleanDataEntry("bool", true) + val entry3 = BinaryDataEntry("blob", ByteStr(Base58.decode("BzWHaQU").get)) + val tx = DataTransaction + .create( + 1, + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + List(entry1, entry2, entry3), + 100000, + 1526911531530L, + Proofs(Seq(ByteStr(Base58.decode("32mNYSefBTrkVngG5REkmmGAVv69ZvNhpbegmnqDReMTmXNyYqbECPgHgXrX2UwyKGLFS45j7xDFyPXjF8jcfw94").get))) + ) + .right + .get + + js shouldEqual tx.json() + } + } diff --git a/src/test/scala/scorex/transaction/IssueTransactionV1Specification.scala b/src/test/scala/scorex/transaction/IssueTransactionV1Specification.scala index 731e7b3ca5e..a5a142c44d5 100644 --- a/src/test/scala/scorex/transaction/IssueTransactionV1Specification.scala +++ b/src/test/scala/scorex/transaction/IssueTransactionV1Specification.scala @@ -1,9 +1,13 @@ package scorex.transaction import com.wavesplatform.TransactionGen +import com.wavesplatform.state.ByteStr import org.scalatest._ import org.scalatest.prop.PropertyChecks -import scorex.transaction.assets.IssueTransaction +import play.api.libs.json.Json +import scorex.account.{PublicKeyAccount} +import scorex.crypto.encode.Base58 +import scorex.transaction.assets.{IssueTransaction, IssueTransactionV1} class IssueTransactionV1Specification extends PropSpec with PropertyChecks with Matchers with TransactionGen { @@ -21,4 +25,41 @@ class IssueTransactionV1Specification extends PropSpec with PropertyChecks with } } + property("JSON format validation") { + val js = Json.parse("""{ + "type": 3, + "id": "9ekQuYn92natMnMq8KqeGK3Nn7cpKd3BvPEGgD6fFyyz", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000000, + "timestamp": 1526287561757, + "version": 1, + "signature": "28kE1uN1pX2bwhzr9UHw5UuB9meTFEDFgeunNgy6nZWpHX4pzkGYotu8DhQ88AdqUG6Yy5wcXgHseKPBUygSgRMJ", + "name": "Gigacoin", + "quantity": 10000000000, + "reissuable": true, + "decimals": 8, + "description": "Gigacoin", + "script":null + } + """) + + val tx = IssueTransactionV1 + .create( + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + "Gigacoin".getBytes, + "Gigacoin".getBytes, + 10000000000L, + 8, + true, + 100000000, + 1526287561757L, + ByteStr(Base58.decode("28kE1uN1pX2bwhzr9UHw5UuB9meTFEDFgeunNgy6nZWpHX4pzkGYotu8DhQ88AdqUG6Yy5wcXgHseKPBUygSgRMJ").get) + ) + .right + .get + + js shouldEqual tx.json() + } + } diff --git a/src/test/scala/scorex/transaction/IssueTransactionV2Specification.scala b/src/test/scala/scorex/transaction/IssueTransactionV2Specification.scala index c9061057162..39db4d1522a 100644 --- a/src/test/scala/scorex/transaction/IssueTransactionV2Specification.scala +++ b/src/test/scala/scorex/transaction/IssueTransactionV2Specification.scala @@ -1,9 +1,12 @@ package scorex.transaction import com.wavesplatform.{TransactionGen, WithDB} -import com.wavesplatform.state.HistoryTest +import com.wavesplatform.state.{ByteStr, HistoryTest} import org.scalatest.{Matchers, PropSpec} import org.scalatest.prop.PropertyChecks +import play.api.libs.json.Json +import scorex.account.PublicKeyAccount +import scorex.crypto.encode.Base58 import scorex.transaction.assets.IssueTransactionV2 class IssueTransactionV2Specification extends PropSpec with PropertyChecks with Matchers with TransactionGen with WithDB with HistoryTest { @@ -24,4 +27,47 @@ class IssueTransactionV2Specification extends PropSpec with PropertyChecks with tx.bytes() shouldEqual recovered.bytes() } } + + property("JSON format validation") { + val js = Json.parse("""{ + "type": 3, + "id": "2ykNAo5JrvNCcL8PtCmc9pTcNtKUy2PjJkrFdRvTfUf4", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000000, + "timestamp": 1526287561757, + "proofs": [ + "43TCfWBa6t2o2ggsD4bU9FpvH3kmDbSBWKE1Z6B5i5Ax5wJaGT2zAvBihSbnSS3AikZLcicVWhUk1bQAMWVzTG5g" + ], + "version": 2, + "name": "Gigacoin", + "quantity": 10000000000, + "reissuable": true, + "decimals": 8, + "description": "Gigacoin", + "script": null + } + """) + + val tx = IssueTransactionV2 + .create( + 2, + 'T', + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + "Gigacoin".getBytes, + "Gigacoin".getBytes, + 10000000000L, + 8, + true, + None, + 100000000, + 1526287561757L, + Proofs(Seq(ByteStr(Base58.decode("43TCfWBa6t2o2ggsD4bU9FpvH3kmDbSBWKE1Z6B5i5Ax5wJaGT2zAvBihSbnSS3AikZLcicVWhUk1bQAMWVzTG5g").get))) + ) + .right + .get + + js shouldEqual tx.json() + } + } diff --git a/src/test/scala/scorex/transaction/LeaseCancelTransactionSpecification.scala b/src/test/scala/scorex/transaction/LeaseCancelTransactionSpecification.scala index 7fbbfcf8c58..fd1951d3d63 100644 --- a/src/test/scala/scorex/transaction/LeaseCancelTransactionSpecification.scala +++ b/src/test/scala/scorex/transaction/LeaseCancelTransactionSpecification.scala @@ -1,9 +1,13 @@ package scorex.transaction import com.wavesplatform.TransactionGen +import com.wavesplatform.state.ByteStr import org.scalatest._ import org.scalatest.prop.PropertyChecks -import scorex.transaction.lease.LeaseCancelTransaction +import play.api.libs.json.Json +import scorex.account.PublicKeyAccount +import scorex.crypto.encode.Base58 +import scorex.transaction.lease.{LeaseCancelTransaction, LeaseCancelTransactionV1, LeaseCancelTransactionV2} class LeaseCancelTransactionSpecification extends PropSpec with PropertyChecks with Matchers with TransactionGen { @@ -28,4 +32,69 @@ class LeaseCancelTransactionSpecification extends PropSpec with PropertyChecks w first.bytes() shouldEqual second.bytes() } + property("JSON format validation for LeaseCancelTransactionV1") { + val js = Json.parse("""{ + "type": 9, + "id": "7hmabbFS8a2z79a29pzZH1s8LHxrsEAnnLjJxNdZ1gGw", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 1000000, + "timestamp": 1526646300260, + "signature": "4T76AXcksn2ixhyMNu4m9UyY54M3HDTw5E2HqUsGV4phogs2vpgBcN5oncu4sbW4U3KU197yfHMxrc3kZ7e6zHG3", + "version": 1, + "leaseId": "EXhjYjy8a1dURbttrGzfcft7cddDnPnoa3vqaBLCTFVY", + "chainId": null + } + """) + + /** + * sender: PublicKeyAccount, leaseId: ByteStr, fee: Long, timestamp: Long, signature: ByteStr + */ + val tx = LeaseCancelTransactionV1 + .create( + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + ByteStr(Base58.decode("EXhjYjy8a1dURbttrGzfcft7cddDnPnoa3vqaBLCTFVY").get), + 1000000, + 1526646300260L, + ByteStr(Base58.decode("4T76AXcksn2ixhyMNu4m9UyY54M3HDTw5E2HqUsGV4phogs2vpgBcN5oncu4sbW4U3KU197yfHMxrc3kZ7e6zHG3").get) + ) + .right + .get + + js shouldEqual tx.json() + } + + property("JSON format validation for LeaseCancelTransactionV2") { + val js = Json.parse("""{ + "type": 9, + "id": "4nvUUiQjTH7D2LFyzaxs8JwaZYZHDggJgq1iP99TvVDM", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 1000000, + "timestamp": 1526646300260, + "proofs": [ + "3h5SQLbCzaLoTHUeoCjXUHB6qhNUfHZjQQVsWTRAgTGMEdK5aeULMVUfDq63J56kkHJiviYTDT92bLGc8ELrUgvi" + ], + "version": 2, + "leaseId": "DJWkQxRyJNqWhq9qSQpK2D4tsrct6eZbjSv3AH4PSha6", + "chainId": 84 + } + """) + + val tx = LeaseCancelTransactionV2 + .create( + 2, + 'T', + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + ByteStr(Base58.decode("DJWkQxRyJNqWhq9qSQpK2D4tsrct6eZbjSv3AH4PSha6").get), + 1000000, + 1526646300260L, + Proofs(Seq(ByteStr(Base58.decode("3h5SQLbCzaLoTHUeoCjXUHB6qhNUfHZjQQVsWTRAgTGMEdK5aeULMVUfDq63J56kkHJiviYTDT92bLGc8ELrUgvi").get))) + ) + .right + .get + + js shouldEqual tx.json() + } + } diff --git a/src/test/scala/scorex/transaction/LeaseTransactionSpecification.scala b/src/test/scala/scorex/transaction/LeaseTransactionSpecification.scala index 1a97b0c556b..8622f1a5656 100644 --- a/src/test/scala/scorex/transaction/LeaseTransactionSpecification.scala +++ b/src/test/scala/scorex/transaction/LeaseTransactionSpecification.scala @@ -1,9 +1,13 @@ package scorex.transaction import com.wavesplatform.TransactionGen +import com.wavesplatform.state.ByteStr import org.scalatest._ import org.scalatest.prop.PropertyChecks -import scorex.transaction.lease.LeaseTransaction +import play.api.libs.json.Json +import scorex.account.{Address, PublicKeyAccount} +import scorex.crypto.encode.Base58 +import scorex.transaction.lease.{LeaseTransaction, LeaseTransactionV1, LeaseTransactionV2} class LeaseTransactionSpecification extends PropSpec with PropertyChecks with Matchers with TransactionGen { @@ -29,4 +33,67 @@ class LeaseTransactionSpecification extends PropSpec with PropertyChecks with Ma first.proofs shouldEqual second.proofs first.bytes() shouldEqual second.bytes() } + + property("JSON format validation for LeaseTransactionV1") { + val js = Json.parse("""{ + "type": 8, + "id": "EXhjYjy8a1dURbttrGzfcft7cddDnPnoa3vqaBLCTFVY", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 1000000, + "timestamp": 1526646300260, + "signature": "iy3TmfbFds7pc9cDDqfjEJhfhVyNtm3GcxoVz8L3kJFvgRPUmiqqKLMeJGYyN12AhaQ6HvE7aF1tFgaAoCCgNJJ", + "version": 1, + "amount": 10000000, + "recipient": "3NCBMxgdghg4tUhEEffSXy11L6hUi6fcBpd" + } + """) + + val tx = LeaseTransactionV1 + .create( + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + 10000000, + 1000000, + 1526646300260L, + Address.fromString("3NCBMxgdghg4tUhEEffSXy11L6hUi6fcBpd").right.get, + ByteStr(Base58.decode("iy3TmfbFds7pc9cDDqfjEJhfhVyNtm3GcxoVz8L3kJFvgRPUmiqqKLMeJGYyN12AhaQ6HvE7aF1tFgaAoCCgNJJ").get) + ) + .right + .get + + js shouldEqual tx.json() + } + + property("JSON format validation for LeaseTransactionV2") { + val js = Json.parse("""{ + "type": 8, + "id": "DJWkQxRyJNqWhq9qSQpK2D4tsrct6eZbjSv3AH4PSha6", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 1000000, + "timestamp": 1526646497465, + "proofs": [ + "5Fr3yLwvfKGDsFLi8A8JbHqToHDojrPbdEGx9mrwbeVWWoiDY5pRqS3rcX1rXC9ud52vuxVdBmGyGk5krcgwFu9q" + ], + "version": 2, + "amount": 10000000, + "recipient": "3NCBMxgdghg4tUhEEffSXy11L6hUi6fcBpd" + } + """) + + val tx = LeaseTransactionV2 + .create( + 2, + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + 10000000, + 1000000, + 1526646497465L, + Address.fromString("3NCBMxgdghg4tUhEEffSXy11L6hUi6fcBpd").right.get, + Proofs(Seq(ByteStr(Base58.decode("5Fr3yLwvfKGDsFLi8A8JbHqToHDojrPbdEGx9mrwbeVWWoiDY5pRqS3rcX1rXC9ud52vuxVdBmGyGk5krcgwFu9q").get))) + ) + .right + .get + + js shouldEqual tx.json() + } } diff --git a/src/test/scala/scorex/transaction/MassTransferTransactionSpecification.scala b/src/test/scala/scorex/transaction/MassTransferTransactionSpecification.scala index 494b66550fa..cdd89475cf4 100644 --- a/src/test/scala/scorex/transaction/MassTransferTransactionSpecification.scala +++ b/src/test/scala/scorex/transaction/MassTransferTransactionSpecification.scala @@ -1,11 +1,15 @@ package scorex.transaction import com.wavesplatform.TransactionGen +import com.wavesplatform.state.ByteStr import org.scalacheck.Arbitrary import org.scalatest._ import org.scalatest.prop.PropertyChecks +import play.api.libs.json.Json +import scorex.account.PublicKeyAccount +import scorex.crypto.encode.Base58 import scorex.transaction.ValidationError.GenericError -import scorex.transaction.transfer.MassTransferTransaction.{MaxTransferCount, ParsedTransfer} +import scorex.transaction.transfer.MassTransferTransaction.{MaxTransferCount, ParsedTransfer, Transfer} import scorex.transaction.transfer._ class MassTransferTransactionSpecification extends PropSpec with PropertyChecks with Matchers with TransactionGen { @@ -74,4 +78,55 @@ class MassTransferTransactionSpecification extends PropSpec with PropertyChecks negativeFeeEi shouldBe Left(ValidationError.InsufficientFee()) } } + + property(testName = "JSON format validation") { + val js = Json.parse("""{ + "type": 11, + "id": "H36CTJc7ztGRZPCrvpNYeagCN1HV1gXqUthsXKdBT3UD", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 200000, + "timestamp": 1518091313964, + "proofs": [ + "FXMNu3ecy5zBjn9b69VtpuYRwxjCbxdkZ3xZpLzB8ZeFDvcgTkmEDrD29wtGYRPtyLS3LPYrL2d5UM6TpFBMUGQ"], + "version": 1, + "assetId": null, + "attachment": "59QuUcqP6p", + "transferCount": 2, + "totalAmount": 300000000, + "transfers": [ + { + "recipient": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "amount": 100000000 + }, + { + "recipient": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "amount": 200000000 + } + ] + } + """) + + val transfers = MassTransferTransaction + .parseTransfersList( + List(Transfer("3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", 100000000L), Transfer("3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", 200000000L))) + .right + .get + + val tx = MassTransferTransaction + .create( + 1, + None, + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + transfers, + 1518091313964L, + 200000, + Base58.decode("59QuUcqP6p").get, + Proofs(Seq(ByteStr(Base58.decode("FXMNu3ecy5zBjn9b69VtpuYRwxjCbxdkZ3xZpLzB8ZeFDvcgTkmEDrD29wtGYRPtyLS3LPYrL2d5UM6TpFBMUGQ").get))) + ) + .right + .get + + js shouldEqual tx.json() + } } diff --git a/src/test/scala/scorex/transaction/ReissueTransactionV1Specification.scala b/src/test/scala/scorex/transaction/ReissueTransactionV1Specification.scala index 9c808e59b4e..1b2350cb0ba 100644 --- a/src/test/scala/scorex/transaction/ReissueTransactionV1Specification.scala +++ b/src/test/scala/scorex/transaction/ReissueTransactionV1Specification.scala @@ -1,9 +1,13 @@ package scorex.transaction import com.wavesplatform.TransactionGen +import com.wavesplatform.state.ByteStr import org.scalatest._ import org.scalatest.prop.PropertyChecks -import scorex.transaction.assets.ReissueTransaction +import play.api.libs.json.Json +import scorex.account.PublicKeyAccount +import scorex.crypto.encode.Base58 +import scorex.transaction.assets.{ReissueTransactionV1, ReissueTransaction} class ReissueTransactionV1Specification extends PropSpec with PropertyChecks with Matchers with TransactionGen { @@ -21,4 +25,37 @@ class ReissueTransactionV1Specification extends PropSpec with PropertyChecks wit } } + property("JSON format validation") { + val js = Json.parse("""{ + "type": 5, + "id": "2y8pNQteNQnY5JWtrZGLUv3tD6GFT6DDzBWttVTwBa2t", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000000, + "timestamp": 1526287561757, + "signature": "3LnRMrjkk7RoV35PTwcdB4yW2rqUqXaKAh8DnPk5tNWABvhVQ9oqdTk3zM8b9AbGtry7WEcQZtevfK92DCFaa6hA", + "version": 1, + "chainId": null, + "assetId": "9ekQuYn92natMnMq8KqeGK3Nn7cpKd3BvPEGgD6fFyyz", + "quantity": 100000000, + "reissuable": true + } + """) + + val tx = ReissueTransactionV1 + .create( + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + ByteStr(Base58.decode("9ekQuYn92natMnMq8KqeGK3Nn7cpKd3BvPEGgD6fFyyz").get), + 100000000L, + true, + 100000000L, + 1526287561757L, + ByteStr(Base58.decode("3LnRMrjkk7RoV35PTwcdB4yW2rqUqXaKAh8DnPk5tNWABvhVQ9oqdTk3zM8b9AbGtry7WEcQZtevfK92DCFaa6hA").get) + ) + .right + .get + + js shouldEqual tx.json() + } + } diff --git a/src/test/scala/scorex/transaction/ReissueTransactionV2Specification.scala b/src/test/scala/scorex/transaction/ReissueTransactionV2Specification.scala new file mode 100644 index 00000000000..a0a2c5b3ad8 --- /dev/null +++ b/src/test/scala/scorex/transaction/ReissueTransactionV2Specification.scala @@ -0,0 +1,51 @@ +package scorex.transaction + +import com.wavesplatform.TransactionGen +import com.wavesplatform.state.ByteStr +import org.scalatest._ +import org.scalatest.prop.PropertyChecks +import play.api.libs.json.Json +import scorex.account.PublicKeyAccount +import scorex.crypto.encode.Base58 +import scorex.transaction.assets.{ReissueTransactionV2} + +class ReissueTransactionV2Specification extends PropSpec with PropertyChecks with Matchers with TransactionGen { + + property("JSON format validation") { + val js = Json.parse("""{ + "type": 5, + "id": "HbQ7gMoDyRxSU6LbLLBVNTbxASaR8rm4Zck6eYvWVUkB", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000000, + "timestamp": 1526287561757, + "proofs": [ + "4DFEtUwJ9gjMQMuEXipv2qK7rnhhWEBqzpC3ZQesW1Kh8D822t62e3cRGWNU3N21r7huWnaty95wj2tZxYSvCfro" + ], + "version": 2, + "chainId": 84, + "assetId": "9ekQuYn92natMnMq8KqeGK3Nn7cpKd3BvPEGgD6fFyyz", + "quantity": 100000000, + "reissuable": true + } + """) + + val tx = ReissueTransactionV2 + .create( + 2, + 'T', + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + ByteStr(Base58.decode("9ekQuYn92natMnMq8KqeGK3Nn7cpKd3BvPEGgD6fFyyz").get), + 100000000L, + true, + 100000000L, + 1526287561757L, + Proofs(Seq(ByteStr(Base58.decode("4DFEtUwJ9gjMQMuEXipv2qK7rnhhWEBqzpC3ZQesW1Kh8D822t62e3cRGWNU3N21r7huWnaty95wj2tZxYSvCfro").get))) + ) + .right + .get + + js shouldEqual tx.json() + } + +} diff --git a/src/test/scala/scorex/transaction/TransferTransactionV1Specification.scala b/src/test/scala/scorex/transaction/TransferTransactionV1Specification.scala index 1b0f78b624d..1b0e69aa772 100644 --- a/src/test/scala/scorex/transaction/TransferTransactionV1Specification.scala +++ b/src/test/scala/scorex/transaction/TransferTransactionV1Specification.scala @@ -1,9 +1,13 @@ package scorex.transaction import com.wavesplatform.TransactionGen +import com.wavesplatform.state.ByteStr import org.scalatest._ import org.scalatest.prop.PropertyChecks +import play.api.libs.json.Json +import scorex.account.{Address, PublicKeyAccount} import scorex.transaction.transfer._ +import scorex.crypto.encode.Base58 class TransferTransactionV1Specification extends PropSpec with PropertyChecks with Matchers with TransactionGen { @@ -30,4 +34,39 @@ class TransferTransactionV1Specification extends PropSpec with PropertyChecks wi } } + property("JSON format validation") { + val js = Json.parse("""{ + "type": 4, + "id": "FLszEaqasJptohmP6zrXodBwjaEYq4jRP2BzdPPjvukk", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000, + "timestamp": 1526552510868, + "signature": "eaV1i3hEiXyYQd6DQY7EnPg9XzpAvB9VA3bnpin2qJe4G36GZXaGnYKCgSf9xiQ61DcAwcBFzjSXh6FwCgazzFz", + "version": 1, + "recipient": "3My3KZgFQ3CrVHgz6vGRt8687sH4oAA1qp8", + "assetId": null, + "feeAssetId": null, + "amount": 1900000, + "attachment": "4t2Xazb2SX" + } + """) + + val tx = TransferTransactionV1 + .create( + None, + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + Address.fromString("3My3KZgFQ3CrVHgz6vGRt8687sH4oAA1qp8").right.get, + 1900000, + 1526552510868L, + None, + 100000, + Base58.decode("4t2Xazb2SX").get, + ByteStr(Base58.decode("eaV1i3hEiXyYQd6DQY7EnPg9XzpAvB9VA3bnpin2qJe4G36GZXaGnYKCgSf9xiQ61DcAwcBFzjSXh6FwCgazzFz").get) + ) + .right + .get + + js shouldEqual tx.json() + } } diff --git a/src/test/scala/scorex/transaction/TransferTransactionV2Specification.scala b/src/test/scala/scorex/transaction/TransferTransactionV2Specification.scala index 6a2aa90ac45..e29b6c301ca 100644 --- a/src/test/scala/scorex/transaction/TransferTransactionV2Specification.scala +++ b/src/test/scala/scorex/transaction/TransferTransactionV2Specification.scala @@ -5,6 +5,9 @@ import com.wavesplatform.state._ import org.scalacheck.Gen import org.scalatest._ import org.scalatest.prop.PropertyChecks +import play.api.libs.json.Json +import scorex.account.{Address, PublicKeyAccount} +import scorex.crypto.encode.Base58 import scorex.transaction.transfer._ class TransferTransactionV2Specification extends PropSpec with PropertyChecks with Matchers with TransactionGen { @@ -46,4 +49,42 @@ class TransferTransactionV2Specification extends PropSpec with PropertyChecks wi first.proofs shouldEqual second.proofs first.bytes() shouldEqual second.bytes() } + + property("JSON format validation") { + val js = Json.parse("""{ + "type": 4, + "id": "2qMiGUpNMuRpeyTnXLa1mLuVP1cYEtxys55cQbDaXd5g", + "sender": "3N5GRqzDBhjVXnCn44baHcz2GoZy5qLxtTh", + "senderPublicKey": "FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z", + "fee": 100000000, + "timestamp": 1526641218066, + "proofs": [ + "4bfDaqBcnK3hT8ywFEFndxtS1DTSYfncUqd4s5Vyaa66PZHawtC73rDswUur6QZu5RpqM7L9NFgBHT1vhCoox4vi" + ], + "version": 2, + "recipient": "3My3KZgFQ3CrVHgz6vGRt8687sH4oAA1qp8", + "assetId": null, + "feeAssetId": null, + "amount": 100000000, + "attachment": "4t2Xazb2SX"} + """) + + val tx = TransferTransactionV2 + .create( + 2, + None, + PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, + Address.fromString("3My3KZgFQ3CrVHgz6vGRt8687sH4oAA1qp8").right.get, + 100000000, + 1526641218066L, + None, + 100000000, + Base58.decode("4t2Xazb2SX").get, + Proofs(Seq(ByteStr(Base58.decode("4bfDaqBcnK3hT8ywFEFndxtS1DTSYfncUqd4s5Vyaa66PZHawtC73rDswUur6QZu5RpqM7L9NFgBHT1vhCoox4vi").get))) + ) + .right + .get + + js shouldEqual tx.json() + } } From 0388d8029c2babd07c5005181bc69af2547877bd Mon Sep 17 00:00:00 2001 From: Nastya Urlapova Date: Mon, 21 May 2018 17:39:06 +0300 Subject: [PATCH 13/52] Update LeaseCancelTransactionSpecification.scala --- .../transaction/LeaseCancelTransactionSpecification.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/scala/scorex/transaction/LeaseCancelTransactionSpecification.scala b/src/test/scala/scorex/transaction/LeaseCancelTransactionSpecification.scala index fd1951d3d63..c3c9a35934a 100644 --- a/src/test/scala/scorex/transaction/LeaseCancelTransactionSpecification.scala +++ b/src/test/scala/scorex/transaction/LeaseCancelTransactionSpecification.scala @@ -47,9 +47,6 @@ class LeaseCancelTransactionSpecification extends PropSpec with PropertyChecks w } """) - /** - * sender: PublicKeyAccount, leaseId: ByteStr, fee: Long, timestamp: Long, signature: ByteStr - */ val tx = LeaseCancelTransactionV1 .create( PublicKeyAccount.fromBase58String("FM5ojNqW7e9cZ9zhPYGkpSP1Pcd8Z3e3MNKYVS5pGJ8Z").right.get, From 7f24304fdb93f62dc27d967407e5225e21b3acbc Mon Sep 17 00:00:00 2001 From: Nastya Urlapova Date: Mon, 21 May 2018 17:48:09 +0300 Subject: [PATCH 14/52] Update DataTransactionSpecification.scala --- .../scala/scorex/transaction/DataTransactionSpecification.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/scorex/transaction/DataTransactionSpecification.scala b/src/test/scala/scorex/transaction/DataTransactionSpecification.scala index d32f09ffba8..f07646c915d 100644 --- a/src/test/scala/scorex/transaction/DataTransactionSpecification.scala +++ b/src/test/scala/scorex/transaction/DataTransactionSpecification.scala @@ -8,7 +8,7 @@ import org.scalacheck.{Arbitrary, Gen} import org.scalatest._ import org.scalatest.prop.PropertyChecks import play.api.libs.json.{Format, Json} -import scorex.account.{Alias, PublicKeyAccount} +import scorex.account.PublicKeyAccount import scorex.api.http.SignedDataRequest import scorex.crypto.encode.Base58 import scorex.transaction.DataTransaction.MaxEntryCount From 5f7b2aae45d81fc8597e1afc4ed680fad578b0d4 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 21 May 2018 17:57:00 +0300 Subject: [PATCH 15/52] NODE-764 Use base64 to encode scripts --- .../SignAndBroadcastApiSuite.scala | 13 +++++++ .../com/wavesplatform/utils/Base64.scala | 9 +++++ .../com/wavesplatform/utils/Base64Test.scala | 36 +++++++++++++++++++ .../com/wavesplatform/state/DataEntry.scala | 10 ++---- .../scala/scorex/api/http/UtilsApiRoute.scala | 17 ++++----- .../http/assets/SignedIssueV2Request.scala | 2 +- .../http/assets/SignedSetScriptRequest.scala | 2 +- .../transaction/TransactionFactory.scala | 4 +-- .../smart/SetScriptTransaction.scala | 3 +- .../transaction/smart/script/Script.scala | 6 ++-- .../wavesplatform/http/UtilsRouteSpec.scala | 10 +++--- 11 files changed, 84 insertions(+), 28 deletions(-) create mode 100644 lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala create mode 100644 lang/jvm/src/test/scala/com/wavesplatform/utils/Base64Test.scala diff --git a/it/src/test/scala/com/wavesplatform/it/sync/transactions/SignAndBroadcastApiSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/transactions/SignAndBroadcastApiSuite.scala index 5d191371308..ee645d4fefc 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/transactions/SignAndBroadcastApiSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/transactions/SignAndBroadcastApiSuite.scala @@ -202,6 +202,19 @@ class SignAndBroadcastApiSuite extends BaseTransactionSuite { ) } + test("/transactions/sign should produce script transaction that is good for /transactions/broadcast") { + signAndBroadcast( + Json.obj( + "type" -> 13, + "version" -> 1, + "sender" -> firstAddress, + "script" -> None, + "fee" -> 100000 + ), + usesProofs = true + ) + } + test("/transactions/sign should produce sponsor transactions that are good for /transactions/broadcast") { for (v <- supportedVersions) { val isProof = Option(v).nonEmpty diff --git a/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala b/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala new file mode 100644 index 00000000000..08c67a8756c --- /dev/null +++ b/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala @@ -0,0 +1,9 @@ +package com.wavesplatform.utils + +import scala.util.Try + +object Base64 { + def encode(input: Array[Byte]): String = "base64:" + new String(java.util.Base64.getEncoder.encode(input)) + + def decode(input: String): Try[Array[Byte]] = Try(java.util.Base64.getDecoder.decode(input.substring(7))) +} diff --git a/lang/jvm/src/test/scala/com/wavesplatform/utils/Base64Test.scala b/lang/jvm/src/test/scala/com/wavesplatform/utils/Base64Test.scala new file mode 100644 index 00000000000..5a5d43c76dc --- /dev/null +++ b/lang/jvm/src/test/scala/com/wavesplatform/utils/Base64Test.scala @@ -0,0 +1,36 @@ +package com.wavesplatform.utils + +import org.scalacheck.Gen +import org.scalatest.{Matchers, PropSpec} +import org.scalatest.prop.PropertyChecks + +class Base64Test extends PropSpec with PropertyChecks with Matchers { + + private val Base64Chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+/" + private val IllegalChars = "!@#$%^&*()_-?/.,<>|\';:`~" + + val illegalGen: Gen[String] = + for { + length <- Gen.chooseNum(100, 1024) + chars <- Gen + .listOfN(length, Gen.oneOf(Base64Chars ++ IllegalChars)) + .filter(_.toSet.intersect(IllegalChars.toSet).nonEmpty) + } yield chars.mkString + + property("handles empty sequences") { + Base64.encode(Array.emptyByteArray) shouldBe "base64:" + val d = Base64.decode("base64:") + d.isSuccess shouldBe true + d.get.length shouldBe 0 + } + + property("decoding fails on illegal characters") { + forAll(illegalGen) { s => + Base64.decode(s).isSuccess shouldBe false + } + } + + property("decoding fails on null") { + Base64.decode(null).isSuccess shouldBe false + } +} diff --git a/src/main/scala/com/wavesplatform/state/DataEntry.scala b/src/main/scala/com/wavesplatform/state/DataEntry.scala index bef0503a258..e65fc739e95 100644 --- a/src/main/scala/com/wavesplatform/state/DataEntry.scala +++ b/src/main/scala/com/wavesplatform/state/DataEntry.scala @@ -4,12 +4,10 @@ import java.nio.charset.StandardCharsets.UTF_8 import com.google.common.primitives.{Longs, Shorts} import com.wavesplatform.state.DataEntry._ +import com.wavesplatform.utils.Base64 import play.api.libs.json._ -import scorex.crypto.encode.Base64 import scorex.serialization.Deser -import scala.util.Try - sealed abstract class DataEntry[T](val key: String, val value: T) { def valueBytes: Array[Byte] @@ -72,9 +70,7 @@ object DataEntry { case JsDefined(JsString("binary")) => jsv \ "value" match { case JsDefined(JsString(enc)) => - if (enc.startsWith("base64:")) - Try(Base64.decode(enc.substring(7))).fold(ex => JsError(ex.getMessage), arr => JsSuccess(BinaryDataEntry(key, ByteStr(arr)))) - else JsError("Base64 encoding expected") + Base64.decode(enc).fold(ex => JsError(ex.getMessage), arr => JsSuccess(BinaryDataEntry(key, ByteStr(arr)))) case _ => JsError("value is missing or not a string") } case JsDefined(JsString("string")) => @@ -108,7 +104,7 @@ case class BooleanDataEntry(override val key: String, override val value: Boolea case class BinaryDataEntry(override val key: String, override val value: ByteStr) extends DataEntry[ByteStr](key, value) { override def valueBytes: Array[Byte] = Type.Binary.id.toByte +: Deser.serializeArray(value.arr) - override def toJson: JsObject = super.toJson + ("type" -> JsString("binary")) + ("value" -> JsString("base64:" + Base64.encode(value.arr))) + override def toJson: JsObject = super.toJson + ("type" -> JsString("binary")) + ("value" -> JsString(Base64.encode(value.arr))) override def valid: Boolean = super.valid && value.arr.length <= MaxValueSize } diff --git a/src/main/scala/scorex/api/http/UtilsApiRoute.scala b/src/main/scala/scorex/api/http/UtilsApiRoute.scala index 7a517064fa2..fb00079c458 100755 --- a/src/main/scala/scorex/api/http/UtilsApiRoute.scala +++ b/src/main/scala/scorex/api/http/UtilsApiRoute.scala @@ -8,8 +8,9 @@ import com.wavesplatform.settings.RestAPISettings import com.wavesplatform.state.diffs.CommonValidation import io.swagger.annotations._ import javax.ws.rs.Path + import play.api.libs.json.Json -import com.wavesplatform.utils.Base58 +import com.wavesplatform.utils.{Base58, Base64} import scorex.transaction.smart.script.{Script, ScriptCompiler} import scorex.utils.Time @@ -30,14 +31,14 @@ case class UtilsApiRoute(timeService: Time, settings: RestAPISettings) extends A } @Path("/script/compile") - @ApiOperation(value = "Compile", notes = "Compiles string code to base58 script representation", httpMethod = "POST") + @ApiOperation(value = "Compile", notes = "Compiles string code to base64 script representation", httpMethod = "POST") @ApiImplicitParams( Array( new ApiImplicitParam(name = "code", required = true, dataType = "string", paramType = "body", value = "Script code") )) @ApiResponses( Array( - new ApiResponse(code = 200, message = "base58 or error") + new ApiResponse(code = 200, message = "base64 or error") )) def compile: Route = path("script" / "compile") { (post & entity(as[String])) { code => @@ -46,7 +47,7 @@ case class UtilsApiRoute(timeService: Time, settings: RestAPISettings) extends A e => Json.obj("error" -> e), { case (script, complexity) => Json.obj( - "script" -> script.bytes().base58, + "script" -> Base64.encode(script.bytes().arr), "complexity" -> complexity, "extraFee" -> CommonValidation.ScriptExtraFee ) @@ -57,20 +58,20 @@ case class UtilsApiRoute(timeService: Time, settings: RestAPISettings) extends A } @Path("/script/estimate") - @ApiOperation(value = "Estimate", notes = "Estimates compiled code in Base58 representation", httpMethod = "POST") + @ApiOperation(value = "Estimate", notes = "Estimates compiled code in Base64 representation", httpMethod = "POST") @ApiImplicitParams( Array( - new ApiImplicitParam(name = "code", required = true, dataType = "string", paramType = "body", value = "A compiled Base58 code") + new ApiImplicitParam(name = "code", required = true, dataType = "string", paramType = "body", value = "A compiled Base64 code") )) @ApiResponses( Array( - new ApiResponse(code = 200, message = "base58 or error") + new ApiResponse(code = 200, message = "base64 or error") )) def estimate: Route = path("script" / "estimate") { (post & entity(as[String])) { code => complete( Script - .fromBase58String(code) + .fromBase64String(code) .left .map(_.m) .flatMap { script => diff --git a/src/main/scala/scorex/api/http/assets/SignedIssueV2Request.scala b/src/main/scala/scorex/api/http/assets/SignedIssueV2Request.scala index cdea2f9748a..74bf2927b0b 100644 --- a/src/main/scala/scorex/api/http/assets/SignedIssueV2Request.scala +++ b/src/main/scala/scorex/api/http/assets/SignedIssueV2Request.scala @@ -44,7 +44,7 @@ case class SignedIssueV2Request(@ApiModelProperty(required = true) _proofs <- Proofs.create(_proofBytes) _script <- script match { case None => Right(None) - case Some(s) => Script.fromBase58String(s).map(Some(_)) + case Some(s) => Script.fromBase64String(s).map(Some(_)) } t <- IssueTransactionV2.create( version, diff --git a/src/main/scala/scorex/api/http/assets/SignedSetScriptRequest.scala b/src/main/scala/scorex/api/http/assets/SignedSetScriptRequest.scala index 129de1bd39d..e4551e8e326 100644 --- a/src/main/scala/scorex/api/http/assets/SignedSetScriptRequest.scala +++ b/src/main/scala/scorex/api/http/assets/SignedSetScriptRequest.scala @@ -32,7 +32,7 @@ case class SignedSetScriptRequest(@ApiModelProperty(required = true) _sender <- PublicKeyAccount.fromBase58String(senderPublicKey) _script <- script match { case None => Right(None) - case Some(s) => Script.fromBase58String(s).map(Some(_)) + case Some(s) => Script.fromBase64String(s).map(Some(_)) } _proofBytes <- proofs.traverse(s => parseBase58(s, "invalid proof", Proofs.MaxProofStringSize)) _proofs <- Proofs.create(_proofBytes) diff --git a/src/main/scala/scorex/transaction/TransactionFactory.scala b/src/main/scala/scorex/transaction/TransactionFactory.scala index cf1efff084f..86e9451cda3 100644 --- a/src/main/scala/scorex/transaction/TransactionFactory.scala +++ b/src/main/scala/scorex/transaction/TransactionFactory.scala @@ -94,7 +94,7 @@ object TransactionFactory { signer <- if (request.sender == signerAddress) Right(sender) else wallet.findPrivateKey(signerAddress) script <- request.script match { case None => Right(None) - case Some(s) => Script.fromBase58String(s).map(Some(_)) + case Some(s) => Script.fromBase64String(s).map(Some(_)) } tx <- SetScriptTransaction.signed( request.version, @@ -115,7 +115,7 @@ object TransactionFactory { signer <- if (request.sender == signerAddress) Right(sender) else wallet.findPrivateKey(signerAddress) s <- request.script match { case None => Right(None) - case Some(s) => Script.fromBase58String(s).map(Some(_)) + case Some(s) => Script.fromBase64String(s).map(Some(_)) } tx <- IssueTransactionV2.signed( version = request.version, diff --git a/src/main/scala/scorex/transaction/smart/SetScriptTransaction.scala b/src/main/scala/scorex/transaction/smart/SetScriptTransaction.scala index 15f05dcbde0..45524e2a65e 100644 --- a/src/main/scala/scorex/transaction/smart/SetScriptTransaction.scala +++ b/src/main/scala/scorex/transaction/smart/SetScriptTransaction.scala @@ -3,6 +3,7 @@ package scorex.transaction.smart import com.google.common.primitives.{Bytes, Longs} import com.wavesplatform.crypto import com.wavesplatform.state._ +import com.wavesplatform.utils.Base64 import monix.eval.Coeval import play.api.libs.json.Json import scorex.account._ @@ -36,7 +37,7 @@ case class SetScriptTransaction private (version: Byte, )) override val assetFee = (None, fee) - override val json = Coeval.evalOnce(jsonBase() ++ Json.obj("version" -> version, "script" -> script.map(_.bytes()))) + override val json = Coeval.evalOnce(jsonBase() ++ Json.obj("version" -> version, "script" -> script.map(s => Base64.encode(s.bytes().arr)))) override val bytes: Coeval[Array[Byte]] = Coeval.evalOnce(Bytes.concat(Array(0: Byte), bodyBytes(), proofs.bytes())) } diff --git a/src/main/scala/scorex/transaction/smart/script/Script.scala b/src/main/scala/scorex/transaction/smart/script/Script.scala index cbbd5b537ca..292f45c579c 100644 --- a/src/main/scala/scorex/transaction/smart/script/Script.scala +++ b/src/main/scala/scorex/transaction/smart/script/Script.scala @@ -5,7 +5,7 @@ import com.wavesplatform.lang.Versioned import com.wavesplatform.lang.v1.compiler.Terms import com.wavesplatform.state.ByteStr import monix.eval.Coeval -import com.wavesplatform.utils.Base58 +import com.wavesplatform.utils.Base64 import scorex.transaction.ValidationError.ScriptParseError trait Script extends Versioned { @@ -25,9 +25,9 @@ object Script { val checksumLength = 4 - def fromBase58String(str: String): Either[ScriptParseError, Script] = + def fromBase64String(str: String): Either[ScriptParseError, Script] = for { - bytes <- Base58.decode(str).toEither.left.map(ex => ScriptParseError(s"Unable to decode base58: ${ex.getMessage}")) + bytes <- Base64.decode(str).toEither.left.map(ex => ScriptParseError(s"Unable to decode base64: ${ex.getMessage}")) script <- ScriptReader.fromBytes(bytes) } yield script diff --git a/src/test/scala/com/wavesplatform/http/UtilsRouteSpec.scala b/src/test/scala/com/wavesplatform/http/UtilsRouteSpec.scala index be418c9dfff..5f4e2cbc782 100644 --- a/src/test/scala/com/wavesplatform/http/UtilsRouteSpec.scala +++ b/src/test/scala/com/wavesplatform/http/UtilsRouteSpec.scala @@ -10,7 +10,7 @@ import org.scalacheck.Gen import org.scalatest.prop.PropertyChecks import play.api.libs.json.{JsObject, JsValue} import scorex.api.http.{TooBigArrayAllocation, UtilsApiRoute} -import com.wavesplatform.utils.Base58 +import com.wavesplatform.utils.{Base58, Base64} import scorex.transaction.smart.script.Script import scorex.transaction.smart.script.v1.ScriptV1 import scorex.utils.Time @@ -35,18 +35,18 @@ class UtilsRouteSpec extends RouteSpec("/utils") with RestAPISettingsHelper with val json = responseAs[JsValue] val expectedScript = ScriptV1(script).explicitGet() - Script.fromBase58String((json \ "script").as[String]) shouldBe Right(expectedScript) + Script.fromBase64String((json \ "script").as[String]) shouldBe Right(expectedScript) (json \ "complexity").as[Long] shouldBe 3 (json \ "extraFee").as[Long] shouldBe CommonValidation.ScriptExtraFee } } routePath("/script/estimate") in { - val base58 = ScriptV1(script).explicitGet().bytes().base58 + val base64 = Base64.encode(ScriptV1(script).explicitGet().bytes().arr) - Post(routePath("/script/estimate"), base58) ~> route ~> check { + Post(routePath("/script/estimate"), base64) ~> route ~> check { val json = responseAs[JsValue] - (json \ "script").as[String] shouldBe base58 + (json \ "script").as[String] shouldBe base64 (json \ "scriptText").as[String] shouldBe "FUNCTION_CALL(FunctionHeader(==,List(LONG, LONG)),List(CONST_LONG(1), CONST_LONG(2)),BOOLEAN)" (json \ "complexity").as[Long] shouldBe 3 (json \ "extraFee").as[Long] shouldBe CommonValidation.ScriptExtraFee From 82cff83dbedb6a2a296cf219f5b2214acd7233e0 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 21 May 2018 18:09:03 +0300 Subject: [PATCH 16/52] NODE-764 Fixed compilation --- src/main/scala/com/wavesplatform/utils/JsonFileStorage.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/com/wavesplatform/utils/JsonFileStorage.scala b/src/main/scala/com/wavesplatform/utils/JsonFileStorage.scala index a45f9eef6a6..8f11ba1250c 100644 --- a/src/main/scala/com/wavesplatform/utils/JsonFileStorage.scala +++ b/src/main/scala/com/wavesplatform/utils/JsonFileStorage.scala @@ -5,7 +5,7 @@ import javax.crypto.Cipher import javax.crypto.spec.SecretKeySpec import play.api.libs.json.{Json, Reads, Writes} -import scorex.crypto.encode.Base64 +import scorex.crypto.encode.{Base64 => ScorexBase64} import scala.io.{BufferedSource, Source} @@ -47,7 +47,7 @@ object JsonFileStorage { private def decrypt(key: SecretKeySpec, encryptedValue: String): String = { val cipher: Cipher = Cipher.getInstance(algorithm) cipher.init(Cipher.DECRYPT_MODE, key) - new String(cipher.doFinal(Base64.decode(encryptedValue))) + new String(cipher.doFinal(ScorexBase64.decode(encryptedValue))) } def save[T](value: T, path: String, key: Option[SecretKeySpec])(implicit w: Writes[T]): Unit = { From 66c12d46ef2a6a584bf4257353d4dbe2aacf14ab Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 21 May 2018 18:37:56 +0300 Subject: [PATCH 17/52] NODE-764 Fixed Wallet --- src/main/scala/com/wavesplatform/utils/JsonFileStorage.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/wavesplatform/utils/JsonFileStorage.scala b/src/main/scala/com/wavesplatform/utils/JsonFileStorage.scala index 8f11ba1250c..799abbe620c 100644 --- a/src/main/scala/com/wavesplatform/utils/JsonFileStorage.scala +++ b/src/main/scala/com/wavesplatform/utils/JsonFileStorage.scala @@ -41,7 +41,7 @@ object JsonFileStorage { private def encrypt(key: SecretKeySpec, value: String): String = { val cipher: Cipher = Cipher.getInstance(algorithm) cipher.init(Cipher.ENCRYPT_MODE, key) - Base64.encode(cipher.doFinal(value.getBytes(encoding))) + ScorexBase64.encode(cipher.doFinal(value.getBytes(encoding))) } private def decrypt(key: SecretKeySpec, encryptedValue: String): String = { From d7c1039f43e9c7243386656c6c4675be1b3331cb Mon Sep 17 00:00:00 2001 From: Alexandr M Date: Mon, 21 May 2018 14:09:04 +0300 Subject: [PATCH 18/52] NODE-129: Fix generationSignature creation, hit and merger conflicts --- .../consensus/PoSCalculator.scala | 8 --- .../wavesplatform/consensus/PoSSelector.scala | 34 +++++---- .../features/BlockchainFeature.scala | 4 +- .../com/wavesplatform/mining/Miner.scala | 4 +- .../scorex/api/http/AddressApiRoute.scala | 7 +- .../consensus/FairPoSCalculatorTest.scala | 72 ------------------- waves-devnet.conf | 16 ++--- 7 files changed, 36 insertions(+), 109 deletions(-) delete mode 100644 src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala diff --git a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala index a0d64fe7734..24312232d0b 100644 --- a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala +++ b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala @@ -24,14 +24,6 @@ trait PoSCalculator { def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long - def validBlockDelay(genSig: Array[Byte], baseTarget: Long, balance: Long): Long = { - calculateDelay(hit(genSig), baseTarget, balance) - } - - def validBlockDelay(genSig: Array[Byte], publicKey: Array[Byte], baseTarget: Long, balance: Long): Long = { - calculateDelay(hit(generatorSignature(genSig, publicKey)), baseTarget, balance) - } - def normalize(value: Long, targetBlockDelaySeconds: Long): Double = value * targetBlockDelaySeconds / (60: Double) diff --git a/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala b/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala index 814485aff49..80205218c95 100644 --- a/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala +++ b/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala @@ -1,15 +1,15 @@ package com.wavesplatform.consensus +import cats.implicits._ import com.wavesplatform.features.BlockchainFeatures import com.wavesplatform.features.FeatureProvider._ import com.wavesplatform.settings.BlockchainSettings -import com.wavesplatform.state._ -import com.wavesplatform.state.{Blockchain, ByteStr} +import com.wavesplatform.state.{Blockchain, ByteStr, _} import scorex.block.Block import scorex.consensus.nxt.NxtLikeConsensusBlockData import scorex.transaction.ValidationError import scorex.transaction.ValidationError.GenericError -import cats.implicits._ + import scala.concurrent.duration.FiniteDuration class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { @@ -26,19 +26,16 @@ class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { greatGrandParentTS: Option[Long], currentTime: Long): Either[ValidationError, NxtLikeConsensusBlockData] = { val bt = baseTarget(targetBlockDelay.toSeconds, height, refBlockBT, refBlockTS, greatGrandParentTS, currentTime) - getBlockForGS(height) + blockchain.lastBlock .map(_.consensusData.generationSignature.arr) - .map(gs => NxtLikeConsensusBlockData(bt, ByteStr(gs))) + .map(gs => NxtLikeConsensusBlockData(bt, ByteStr(pos(height).generatorSignature(gs, accountPublicKey)))) .toRight(GenericError("No blocks in blockchain")) } def getValidBlockDelay(height: Int, accountPublicKey: Array[Byte], refBlockBT: Long, balance: Long): Either[ValidationError, Long] = { val pc = pos(height) - getBlockForGS(height) - .map(_.consensusData.generationSignature.arr) - .map(pc.generatorSignature(_, accountPublicKey)) - .map(pc.hit) + getHit(height, accountPublicKey) .map(pc.calculateDelay(_, refBlockBT, balance)) .toRight(GenericError("No blocks in blockchain")) } @@ -51,8 +48,8 @@ class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { } def validateGeneratorSignature(height: Int, block: Block): Either[ValidationError, Unit] = { - getBlockForGS(height) - .map(_.consensusData.generationSignature.arr) + blockchain.lastBlock + .map(b => pos(height).generatorSignature(b.consensusData.generationSignature.arr, block.signerData.generator.publicKey)) .toRight(GenericError("No blocks in blockchain T.T")) .ensure(GenericError("Generation signatures doesnot match"))(_ sameElements block.consensusData.generationSignature.arr) .map(_ => ()) @@ -78,9 +75,16 @@ class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { ) } - private def getBlockForGS(height: Int): Option[Block] = { - if (fairPosActivated(height)) blockchain.blockAt(height - 100) orElse blockchain.lastBlock - else blockchain.lastBlock + private def getHit(height: Int, accountPublicKey: Array[Byte]): Option[BigInt] = { + val blockForHit = + if (fairPosActivated(height) && height > 100) blockchain.blockAt(height - 100) + else blockchain.lastBlock + + blockForHit.map(b => { + val p = pos(height) + val gs = p.generatorSignature(b.consensusData.generationSignature.arr, accountPublicKey) + p.hit(gs) + }) } def baseTarget(targetBlockDelaySeconds: Long, @@ -89,7 +93,7 @@ class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { parentTimestamp: Long, maybeGreatGrandParentTimestamp: Option[Long], timestamp: Long): Long = { - pos(blockchain.height) + pos(prevHeight) .baseTarget( targetBlockDelaySeconds, prevHeight, diff --git a/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala b/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala index 8c9d3e31f8a..c2215b53d4a 100644 --- a/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala +++ b/src/main/scala/com/wavesplatform/features/BlockchainFeature.scala @@ -11,6 +11,7 @@ object BlockchainFeatures { val DataTransaction = BlockchainFeature(5, "Data Transaction") val BurnAnyTokens = BlockchainFeature(6, "Burn Any Tokens") val FeeSponsorship = BlockchainFeature(7, "Fee Sponsorship") + val FairPoS = BlockchainFeature(8, "Fair PoS") private val dict = Seq( SmallerMinimalGeneratingBalance, @@ -19,7 +20,8 @@ object BlockchainFeatures { SmartAccounts, DataTransaction, BurnAnyTokens, - FeeSponsorship + FeeSponsorship, + FairPoS ).map(f => f.id -> f).toMap val implemented: Set[Short] = dict.keySet diff --git a/src/main/scala/com/wavesplatform/mining/Miner.scala b/src/main/scala/com/wavesplatform/mining/Miner.scala index a3c1092a492..c08096bb3a6 100644 --- a/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -294,8 +294,8 @@ class MinerImpl(allChannels: ChannelGroup, private def generateBlockTask(account: PrivateKeyAccount): Task[Unit] = { { - val height = blockchainUpdater.height - val blockForHit = (blockchainUpdater.blockAt(height - 100) orElse blockchainUpdater.lastBlock).get + val height = blockchainUpdater.height + val lastBlock = blockchainUpdater.lastBlock.get for { _ <- checkAge(height, blockchainUpdater.lastBlockTimestamp.get) _ <- checkScript(account) diff --git a/src/main/scala/scorex/api/http/AddressApiRoute.scala b/src/main/scala/scorex/api/http/AddressApiRoute.scala index 3652f42fc41..41eb39f5c2d 100644 --- a/src/main/scala/scorex/api/http/AddressApiRoute.scala +++ b/src/main/scala/scorex/api/http/AddressApiRoute.scala @@ -4,10 +4,12 @@ import java.nio.charset.StandardCharsets import akka.http.scaladsl.marshalling.ToResponseMarshallable import akka.http.scaladsl.server.Route +import com.wavesplatform.consensus.GeneratingBalanceProvider import com.wavesplatform.crypto import com.wavesplatform.settings.{FunctionalitySettings, RestAPISettings} import com.wavesplatform.state.Blockchain import com.wavesplatform.state.diffs.CommonValidation +import com.wavesplatform.utils.Base58 import com.wavesplatform.utx.UtxPool import io.netty.channel.group.ChannelGroup import io.swagger.annotations._ @@ -15,10 +17,9 @@ import javax.ws.rs.Path import play.api.libs.json._ import scorex.BroadcastRoute import scorex.account.{Address, PublicKeyAccount} -import com.wavesplatform.utils.Base58 import scorex.transaction.ValidationError.GenericError import scorex.transaction.smart.script.ScriptCompiler -import scorex.transaction.{PoSCalc, TransactionFactory, ValidationError} +import scorex.transaction.{TransactionFactory, ValidationError} import scorex.utils.Time import scorex.wallet.Wallet @@ -358,7 +359,7 @@ case class AddressApiRoute(settings: RestAPISettings, BalanceDetails( account.address, portfolio.balance, - PoSCalc.generatingBalance(blockchain, functionalitySettings, account, blockchain.height), + GeneratingBalanceProvider.balance(blockchain, functionalitySettings, blockchain.height, account), portfolio.balance - portfolio.lease.out, portfolio.effectiveBalance ) diff --git a/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala b/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala deleted file mode 100644 index 35c63e8e680..00000000000 --- a/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.wavesplatform.consensus - -import org.scalatest.{Matchers, PropSpec} -import scala.util.Random - -class FairPoSCalculatorTest extends PropSpec with Matchers { - - val pos = FairPoSCalculator - - case class Block(height: Int, baseTarget: Long, timestamp: Long, delay: Long) - - def generationSignature: Array[Byte] = { - val arr = new Array[Byte](32) - Random.nextBytes(arr) - arr - } - - property("Correct consensus parameters of blocks generated with FairPoS") { - - val balance = 50000000L * 100000000L - - val blockDelaySeconds = 60 - - val defaultBaseTarget = 100L - - val first = Block(0, defaultBaseTarget, System.currentTimeMillis(), 0) - - val chain = (1 to 10000 foldLeft List(first))((acc, _) => { - acc match { - case last :: _ => - val delay = pos.validBlockDelay(generationSignature, last.baseTarget, balance) - val bt = pos.baseTarget( - blockDelaySeconds, - last.height + 1, - last.baseTarget, - last.timestamp, - if (acc.isDefinedAt(2)) Some(acc(2).timestamp) else None, - last.timestamp + delay - ) - - Block( - last.height + 1, - bt, - last.timestamp + delay, - delay - ) :: acc - - case _ => ??? - } - - }).reverse - - val maxBT = chain.maxBy(_.baseTarget).baseTarget - val avgBT = chain.map(_.baseTarget).sum / chain.length - val minBT = chain.minBy(_.baseTarget).baseTarget - - val maxDelay = chain.tail.maxBy(_.delay).delay - val avgDelay = chain.tail.map(_.delay).sum / (chain.length - 1) - val minDelay = chain.tail.minBy(_.delay).delay - - print( - s""" - |BT: $minBT $avgBT $maxBT - |Delay: $minDelay $avgDelay $maxDelay - """.stripMargin - ) - - assert(avgDelay < 80000 && avgDelay > 40000) - assert(avgBT < 200 && avgBT > 20) - } - -} diff --git a/waves-devnet.conf b/waves-devnet.conf index 2af84dceab1..26caf31945e 100644 --- a/waves-devnet.conf +++ b/waves-devnet.conf @@ -27,17 +27,17 @@ waves { genesis { average-block-delay: 60000ms initial-base-target: 100 - timestamp: 1500635421931 - block-timestamp: 1500635421931 - signature: "GxifkzjW43Cg3xjpNjk5EwhVh5q9EN82WQpGMfNY33c1eCmLFtZGHARwRZLaSZaYss7iTt9yavTBWcXqagCBXii" + timestamp: 1489352400000 + block-timestamp: 1489352400000 + signature: "5ASUNefZs2dLRroid7LPS24PL85K5Y6WZqA1bfQGCHxkfhLK39jHDUpmFzELKQ66AHCm7ZhQVwpF6G95tat3xfpr" initial-balance: 10000000000000000 transactions = [ - {recipient: "3FR3MyuXumwBj1bLC8xnW38iHtwm9Ugdm8K", amount: 1500000000000000}, - {recipient: "3FgScYB6MNdnN8m4xXddQe1Bjkwmd3U7YtM", amount: 1500000000000000}, - {recipient: "3FWXhvWq2r8m54MmCEZ3YZkLg2qUdGWbU3V", amount: 1500000000000000}, - {recipient: "3FkBWsgT9T3snZ4ZpzzQCJWQngJBLdDEPfU", amount: 1500000000000000}, + {recipient: "3FR3MyuXumwBj1bLC8xnW38iHtwm9Ugdm8K", amount: 2500000000000000}, + {recipient: "3FgScYB6MNdnN8m4xXddQe1Bjkwmd3U7YtM", amount: 200000000000000}, + {recipient: "3FWXhvWq2r8m54MmCEZ3YZkLg2qUdGWbU3V", amount: 1000000000000000}, + {recipient: "3FkBWsgT9T3snZ4ZpzzQCJWQngJBLdDEPfU", amount: 500000000000000}, {recipient: "3FeeqPbaEUQ8h3eQ4ZX9WcqzqskGbfTqM2a", amount: 1500000000000000}, - {recipient: "3FcSgww3tKZ7feQVmcnPFmRxsjqBodYz63x", amount: 1500000000000000} + {recipient: "3FcSgww3tKZ7feQVmcnPFmRxsjqBodYz63x", amount: 2000000000000000} ] } } From 1c5de00ed82bc07872b87340588c4f14c278247c Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 21 May 2018 19:35:02 +0300 Subject: [PATCH 19/52] NODE-764 Better error reporting, more test cases --- .../com/wavesplatform/it/sync/DataTransactionSuite.scala | 5 ++++- .../wavesplatform/it/sync/SetScriptTransactionSuite.scala | 6 +++++- .../jvm/src/main/scala/com/wavesplatform/utils/Base64.scala | 5 ++++- src/main/scala/scorex/api/http/AddressApiRoute.scala | 5 +++-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/it/src/test/scala/com/wavesplatform/it/sync/DataTransactionSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/DataTransactionSuite.scala index d9382bd4ad5..7c65a967749 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/DataTransactionSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/DataTransactionSuite.scala @@ -144,6 +144,9 @@ class DataTransactionSuite extends BaseTransactionSuite { sender.getData(secondAddress).equals(dataAllTypes) notMiner.assertBalances(secondAddress, balance2 - fee, eff2 - fee) + + val json = Json.parse(sender.get(s"/transactions/info/$txId").getResponseBody) + ((json \ "data")(2) \ "value").as[String].startsWith("base64:") shouldBe true } test("queries for nonexistent data") { @@ -218,7 +221,7 @@ class DataTransactionSuite extends BaseTransactionSuite { "Illegal base64 character") assertBadRequestAndResponse(sender.postJson("/addresses/data", request(notValidBlobValue + ("value" -> JsString("yomp")))), - "Base64 encoding expected") + "base64:chars expected") } test("transaction requires a valid proof") { diff --git a/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala index 3128d70c1e6..aef82387cfc 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala @@ -9,7 +9,7 @@ import com.wavesplatform.lang.v1.parser.Parser import com.wavesplatform.state._ import com.wavesplatform.utils.dummyTypeCheckerContext import org.scalatest.CancelAfterFailure -import play.api.libs.json.JsNumber +import play.api.libs.json.{JsNumber, Json} import scorex.account.PrivateKeyAccount import scorex.transaction.Proofs import scorex.transaction.transfer._ @@ -85,6 +85,10 @@ class SetScriptTransactionSuite extends BaseTransactionSuite with CancelAfterFai acc0ScriptInfo.script.isEmpty shouldBe false acc0ScriptInfo.scriptText.isEmpty shouldBe false + acc0ScriptInfo.script.get.startsWith("base64:") shouldBe true + + val json = Json.parse(sender.get(s"/transactions/info/$setScriptId").getResponseBody) + (json \ "script").as[String].startsWith("base64:") shouldBe true } test("can't send from acc0 using old pk") { diff --git a/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala b/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala index 08c67a8756c..426201cfea7 100644 --- a/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala +++ b/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala @@ -5,5 +5,8 @@ import scala.util.Try object Base64 { def encode(input: Array[Byte]): String = "base64:" + new String(java.util.Base64.getEncoder.encode(input)) - def decode(input: String): Try[Array[Byte]] = Try(java.util.Base64.getDecoder.decode(input.substring(7))) + def decode(input: String): Try[Array[Byte]] = Try { + if (input.length < 7) throw new IllegalArgumentException("String of the form base64:chars expected") + else java.util.Base64.getDecoder.decode(input.substring(7)) + } } diff --git a/src/main/scala/scorex/api/http/AddressApiRoute.scala b/src/main/scala/scorex/api/http/AddressApiRoute.scala index 3652f42fc41..d7fde86c563 100644 --- a/src/main/scala/scorex/api/http/AddressApiRoute.scala +++ b/src/main/scala/scorex/api/http/AddressApiRoute.scala @@ -12,10 +12,11 @@ import com.wavesplatform.utx.UtxPool import io.netty.channel.group.ChannelGroup import io.swagger.annotations._ import javax.ws.rs.Path + import play.api.libs.json._ import scorex.BroadcastRoute import scorex.account.{Address, PublicKeyAccount} -import com.wavesplatform.utils.Base58 +import com.wavesplatform.utils.{Base58, Base64} import scorex.transaction.ValidationError.GenericError import scorex.transaction.smart.script.ScriptCompiler import scorex.transaction.{PoSCalc, TransactionFactory, ValidationError} @@ -371,7 +372,7 @@ case class AddressApiRoute(settings: RestAPISettings, } yield AddressScriptInfo( address = account.address, - script = script.map(_.bytes().base58), + script = script.map(s => Base64.encode(s.bytes().arr)), scriptText = script.map(_.text), complexity = complexity, extraFee = if (script.isEmpty) 0 else CommonValidation.ScriptExtraFee From 5a0fb78d7591351b8ffb0d36f81496113adcc3f2 Mon Sep 17 00:00:00 2001 From: peterz Date: Mon, 21 May 2018 19:41:12 +0300 Subject: [PATCH 20/52] NODE-764 Test fix --- src/test/scala/com/wavesplatform/http/AddressRouteSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/com/wavesplatform/http/AddressRouteSpec.scala b/src/test/scala/com/wavesplatform/http/AddressRouteSpec.scala index dda35c60d5b..2a6decc15bc 100644 --- a/src/test/scala/com/wavesplatform/http/AddressRouteSpec.scala +++ b/src/test/scala/com/wavesplatform/http/AddressRouteSpec.scala @@ -152,7 +152,7 @@ class AddressRouteSpec Get(routePath(s"/scriptInfo/${allAddresses(1)}")) ~> route ~> check { val response = responseAs[JsObject] (response \ "address").as[String] shouldBe allAddresses(1) - (response \ "script").as[String] shouldBe "We8Dksx" + (response \ "script").as[String] shouldBe "base64:AQa3b8tH" (response \ "scriptText").as[String] shouldBe "TRUE" (response \ "complexity").as[Long] shouldBe 1 (response \ "extraFee").as[Long] shouldBe CommonValidation.ScriptExtraFee From ea7e5916c3bb7606ed86c6ced6994b67ec7d32d8 Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Mon, 21 May 2018 21:34:42 +0300 Subject: [PATCH 21/52] Fix tests. --- lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala | 2 ++ .../main/scala/com/wavesplatform/lang/v1/parser/Parser.scala | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala index 990c18b0593..b54c993f86e 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala @@ -38,6 +38,8 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG forAll(testGen) { case ((expr, str)) => + println(expr) + println(play.api.libs.json.Json.toJson(str.filter(e => !"\r\t".contains(e)))) parse(str) shouldBe expr } } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala index 175024abd5b..50a52f2ede6 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala @@ -18,7 +18,7 @@ object Parser { import White._ import fastparse.noApi._ private val Base58Chars = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" - private val keywords = Set("let", "base58", "true", "false", "if", "then", "else", "match", "case") + val keywords = Set("let", "base58", "true", "false", "if", "then", "else", "match", "case") private val lowerChar = CharIn('a' to 'z') private val upperChar = CharIn('A' to 'Z') From a370f03ccc76fa1e335e11fca5e8b9d35afe55fe Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Tue, 22 May 2018 00:02:00 +0300 Subject: [PATCH 22/52] Remove debbugging. --- lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala index b54c993f86e..990c18b0593 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala @@ -38,8 +38,6 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG forAll(testGen) { case ((expr, str)) => - println(expr) - println(play.api.libs.json.Json.toJson(str.filter(e => !"\r\t".contains(e)))) parse(str) shouldBe expr } } From 7f21cf96c92dbcf08a3a3f96f72fa378db9b3e5b Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Tue, 22 May 2018 00:05:51 +0300 Subject: [PATCH 23/52] Fix test. --- .../scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index 67741a88cd5..97845cce778 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -1,5 +1,6 @@ package com.wavesplatform.lang.v1.testing +import com.wavesplatform.lang.v1.parser.Parser.keywords import com.wavesplatform.lang.v1.parser.BinaryOperation import com.wavesplatform.lang.v1.parser.Expressions._ import com.wavesplatform.lang.v1.parser.BinaryOperation._ @@ -115,7 +116,7 @@ trait ScriptGen { } yield LET(name, value) def REFgen: Gen[EXPR] = - Gen.identifier.map(REF) + Gen.identifier.filter(!keywords(_)).map(REF) def BLOCKgen(gas: Int): Gen[EXPR] = for { From 2a05d362c64df4dd4eb43e216db88310ca85c740 Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 22 May 2018 12:27:06 +0300 Subject: [PATCH 24/52] NODE-764 Introduced ByteStr.base64 --- .../it/async/SmartTransactionsConstraintsSuite.scala | 2 +- src/main/scala/com/wavesplatform/state/ByteStr.scala | 4 +++- src/main/scala/com/wavesplatform/state/DataEntry.scala | 2 +- src/main/scala/scorex/api/http/AddressApiRoute.scala | 7 +++---- src/main/scala/scorex/api/http/UtilsApiRoute.scala | 7 +++---- .../scorex/transaction/smart/SetScriptTransaction.scala | 3 +-- src/test/scala/com/wavesplatform/http/UtilsRouteSpec.scala | 4 ++-- 7 files changed, 14 insertions(+), 15 deletions(-) diff --git a/it/src/test/scala/com/wavesplatform/it/async/SmartTransactionsConstraintsSuite.scala b/it/src/test/scala/com/wavesplatform/it/async/SmartTransactionsConstraintsSuite.scala index 335e5433904..b82ee56ff92 100644 --- a/it/src/test/scala/com/wavesplatform/it/async/SmartTransactionsConstraintsSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/async/SmartTransactionsConstraintsSuite.scala @@ -93,7 +93,7 @@ class SmartTransactionsConstraintsSuite extends FreeSpec with Matchers with Tran private def toRequest(tx: SetScriptTransaction): SignedSetScriptRequest = SignedSetScriptRequest( version = tx.version, senderPublicKey = Base58.encode(tx.sender.publicKey), - script = tx.script.map(_.bytes().base58), + script = tx.script.map(_.bytes().base64), fee = tx.fee, timestamp = tx.timestamp, proofs = tx.proofs.proofs.map(_.base58)(collection.breakOut) diff --git a/src/main/scala/com/wavesplatform/state/ByteStr.scala b/src/main/scala/com/wavesplatform/state/ByteStr.scala index 9a5ef52eb30..f6d5d939e88 100644 --- a/src/main/scala/com/wavesplatform/state/ByteStr.scala +++ b/src/main/scala/com/wavesplatform/state/ByteStr.scala @@ -1,7 +1,7 @@ package com.wavesplatform.state import play.api.libs.json._ -import com.wavesplatform.utils.Base58 +import com.wavesplatform.utils.{Base58, Base64} import scala.util.Try @@ -15,6 +15,8 @@ case class ByteStr(arr: Array[Byte]) { lazy val base58: String = Base58.encode(arr) + lazy val base64: String = Base64.encode(arr) + lazy val trim: String = base58.toString.take(7) + "..." override lazy val toString: String = base58 diff --git a/src/main/scala/com/wavesplatform/state/DataEntry.scala b/src/main/scala/com/wavesplatform/state/DataEntry.scala index e65fc739e95..b66ee4a77cd 100644 --- a/src/main/scala/com/wavesplatform/state/DataEntry.scala +++ b/src/main/scala/com/wavesplatform/state/DataEntry.scala @@ -104,7 +104,7 @@ case class BooleanDataEntry(override val key: String, override val value: Boolea case class BinaryDataEntry(override val key: String, override val value: ByteStr) extends DataEntry[ByteStr](key, value) { override def valueBytes: Array[Byte] = Type.Binary.id.toByte +: Deser.serializeArray(value.arr) - override def toJson: JsObject = super.toJson + ("type" -> JsString("binary")) + ("value" -> JsString(Base64.encode(value.arr))) + override def toJson: JsObject = super.toJson + ("type" -> JsString("binary")) + ("value" -> JsString(value.base64)) override def valid: Boolean = super.valid && value.arr.length <= MaxValueSize } diff --git a/src/main/scala/scorex/api/http/AddressApiRoute.scala b/src/main/scala/scorex/api/http/AddressApiRoute.scala index d7fde86c563..0fd027acd09 100644 --- a/src/main/scala/scorex/api/http/AddressApiRoute.scala +++ b/src/main/scala/scorex/api/http/AddressApiRoute.scala @@ -1,6 +1,7 @@ package scorex.api.http import java.nio.charset.StandardCharsets +import javax.ws.rs.Path import akka.http.scaladsl.marshalling.ToResponseMarshallable import akka.http.scaladsl.server.Route @@ -8,15 +9,13 @@ import com.wavesplatform.crypto import com.wavesplatform.settings.{FunctionalitySettings, RestAPISettings} import com.wavesplatform.state.Blockchain import com.wavesplatform.state.diffs.CommonValidation +import com.wavesplatform.utils.Base58 import com.wavesplatform.utx.UtxPool import io.netty.channel.group.ChannelGroup import io.swagger.annotations._ -import javax.ws.rs.Path - import play.api.libs.json._ import scorex.BroadcastRoute import scorex.account.{Address, PublicKeyAccount} -import com.wavesplatform.utils.{Base58, Base64} import scorex.transaction.ValidationError.GenericError import scorex.transaction.smart.script.ScriptCompiler import scorex.transaction.{PoSCalc, TransactionFactory, ValidationError} @@ -372,7 +371,7 @@ case class AddressApiRoute(settings: RestAPISettings, } yield AddressScriptInfo( address = account.address, - script = script.map(s => Base64.encode(s.bytes().arr)), + script = script.map(_.bytes().base64), scriptText = script.map(_.text), complexity = complexity, extraFee = if (script.isEmpty) 0 else CommonValidation.ScriptExtraFee diff --git a/src/main/scala/scorex/api/http/UtilsApiRoute.scala b/src/main/scala/scorex/api/http/UtilsApiRoute.scala index fb00079c458..fe5b759a4ca 100755 --- a/src/main/scala/scorex/api/http/UtilsApiRoute.scala +++ b/src/main/scala/scorex/api/http/UtilsApiRoute.scala @@ -1,16 +1,15 @@ package scorex.api.http import java.security.SecureRandom +import javax.ws.rs.Path import akka.http.scaladsl.server.Route import com.wavesplatform.crypto import com.wavesplatform.settings.RestAPISettings import com.wavesplatform.state.diffs.CommonValidation +import com.wavesplatform.utils.Base58 import io.swagger.annotations._ -import javax.ws.rs.Path - import play.api.libs.json.Json -import com.wavesplatform.utils.{Base58, Base64} import scorex.transaction.smart.script.{Script, ScriptCompiler} import scorex.utils.Time @@ -47,7 +46,7 @@ case class UtilsApiRoute(timeService: Time, settings: RestAPISettings) extends A e => Json.obj("error" -> e), { case (script, complexity) => Json.obj( - "script" -> Base64.encode(script.bytes().arr), + "script" -> script.bytes().base64, "complexity" -> complexity, "extraFee" -> CommonValidation.ScriptExtraFee ) diff --git a/src/main/scala/scorex/transaction/smart/SetScriptTransaction.scala b/src/main/scala/scorex/transaction/smart/SetScriptTransaction.scala index 45524e2a65e..616aed5a98c 100644 --- a/src/main/scala/scorex/transaction/smart/SetScriptTransaction.scala +++ b/src/main/scala/scorex/transaction/smart/SetScriptTransaction.scala @@ -3,7 +3,6 @@ package scorex.transaction.smart import com.google.common.primitives.{Bytes, Longs} import com.wavesplatform.crypto import com.wavesplatform.state._ -import com.wavesplatform.utils.Base64 import monix.eval.Coeval import play.api.libs.json.Json import scorex.account._ @@ -37,7 +36,7 @@ case class SetScriptTransaction private (version: Byte, )) override val assetFee = (None, fee) - override val json = Coeval.evalOnce(jsonBase() ++ Json.obj("version" -> version, "script" -> script.map(s => Base64.encode(s.bytes().arr)))) + override val json = Coeval.evalOnce(jsonBase() ++ Json.obj("version" -> version, "script" -> script.map(_.bytes().base64))) override val bytes: Coeval[Array[Byte]] = Coeval.evalOnce(Bytes.concat(Array(0: Byte), bodyBytes(), proofs.bytes())) } diff --git a/src/test/scala/com/wavesplatform/http/UtilsRouteSpec.scala b/src/test/scala/com/wavesplatform/http/UtilsRouteSpec.scala index 5f4e2cbc782..411addad9e1 100644 --- a/src/test/scala/com/wavesplatform/http/UtilsRouteSpec.scala +++ b/src/test/scala/com/wavesplatform/http/UtilsRouteSpec.scala @@ -10,7 +10,7 @@ import org.scalacheck.Gen import org.scalatest.prop.PropertyChecks import play.api.libs.json.{JsObject, JsValue} import scorex.api.http.{TooBigArrayAllocation, UtilsApiRoute} -import com.wavesplatform.utils.{Base58, Base64} +import com.wavesplatform.utils.Base58 import scorex.transaction.smart.script.Script import scorex.transaction.smart.script.v1.ScriptV1 import scorex.utils.Time @@ -42,7 +42,7 @@ class UtilsRouteSpec extends RouteSpec("/utils") with RestAPISettingsHelper with } routePath("/script/estimate") in { - val base64 = Base64.encode(ScriptV1(script).explicitGet().bytes().arr) + val base64 = ScriptV1(script).explicitGet().bytes().base64 Post(routePath("/script/estimate"), base64) ~> route ~> check { val json = responseAs[JsValue] From e3c4f469150f68d9473eb10aed2a9af8bc6b68bb Mon Sep 17 00:00:00 2001 From: Sergey Nazarov Date: Fri, 18 May 2018 19:08:33 +0300 Subject: [PATCH 25/52] NODE-748 Asset Distribution speedup --- .../wavesplatform/it/api/AsyncHttpApi.scala | 2 +- .../matcher/OrderExclusionTestSuite.scala | 45 +- .../scala/com/wavesplatform/Application.scala | 2 +- .../scala/com/wavesplatform/Explorer.scala | 117 +++- .../com/wavesplatform/database/Keys.scala | 113 +++ .../database/LevelDBWriter.scala | 644 +++++------------- .../com/wavesplatform/database/package.scala | 174 ++++- .../com/wavesplatform/state/Blockchain.scala | 2 +- .../state/BlockchainUpdaterImpl.scala | 12 +- .../state/appender/BlockAppender.scala | 16 +- .../state/appender/package.scala | 2 +- .../com/wavesplatform/state/package.scala | 2 +- .../state/reader/CompositeBlockchain.scala | 15 +- src/main/scala/scorex/account/Address.scala | 13 +- .../scorex/api/http/BlocksApiRoute.scala | 12 +- .../api/http/assets/AssetsApiRoute.scala | 2 +- .../transaction/BlockchainUpdater.scala | 2 + .../scorex/waves/http/DebugApiRoute.scala | 4 +- 18 files changed, 636 insertions(+), 543 deletions(-) create mode 100644 src/main/scala/com/wavesplatform/database/Keys.scala diff --git a/it/src/main/scala/com/wavesplatform/it/api/AsyncHttpApi.scala b/it/src/main/scala/com/wavesplatform/it/api/AsyncHttpApi.scala index 5309f9bc4e8..3a4b8e94fa5 100644 --- a/it/src/main/scala/com/wavesplatform/it/api/AsyncHttpApi.scala +++ b/it/src/main/scala/com/wavesplatform/it/api/AsyncHttpApi.scala @@ -236,7 +236,7 @@ object AsyncHttpApi extends Assertions { 100.millis ) - def waitForHeight(expectedHeight: Int): Future[Int] = waitFor[Int](s"height >= $expectedHeight")(_.height, h => h >= expectedHeight, 1.second) + def waitForHeight(expectedHeight: Int): Future[Int] = waitFor[Int](s"height >= $expectedHeight")(_.height, h => h >= expectedHeight, 5.seconds) def transactionInfo(txId: String): Future[TransactionInfo] = get(s"/transactions/info/$txId").as[TransactionInfo] diff --git a/it/src/test/scala/com/wavesplatform/it/async/matcher/OrderExclusionTestSuite.scala b/it/src/test/scala/com/wavesplatform/it/async/matcher/OrderExclusionTestSuite.scala index d6b68bd3cf2..ddcd05c3409 100644 --- a/it/src/test/scala/com/wavesplatform/it/async/matcher/OrderExclusionTestSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/async/matcher/OrderExclusionTestSuite.scala @@ -87,7 +87,7 @@ class OrderExclusionTestSuite val signature = ByteStr(crypto.sign(privateKey, pk ++ Longs.toByteArray(ts))) val orderhistory = Await.result(matcherNode.getOrderbookByPublicKey(node.publicKeyStr, ts, signature), 1.minute) - orderhistory.seq(0).status + orderhistory.head.status } } @@ -98,28 +98,27 @@ object OrderExclusionTestSuite { import NodeConfigs.Default private val matcherConfig = ConfigFactory.parseString(s""" - |waves.matcher { - | enable=yes - | account="3Hm3LGoNPmw1VTZ3eRA2pAfeQPhnaBm6YFC" - | bind-address="0.0.0.0" - | order-match-tx-fee = 300000 - | blacklisted-assets = [$ForbiddenAssetId] - | order-cleanup-interval = 20s - |} - |waves.rest-api { + |waves { + | matcher { + | enable = yes + | account = 3HmFkAoQRs4Y3PE2uR6ohN7wS4VqPBGKv7k + | bind-address = "0.0.0.0" + | order-match-tx-fee = 300000 + | blacklisted-assets = [$ForbiddenAssetId] + | order-cleanup-interval = 20s + | } + | rest-api { | enable = yes | api-key-hash = 7L6GpLHhA5KyJTAVc8WFHwEcyTY8fC8rRbyMCiFnM4i - |} - |waves.miner.enable=no - """.stripMargin) + | } + | miner.enable=no + |}""".stripMargin) private val nonGeneratingPeersConfig = ConfigFactory.parseString( - """ - |waves.matcher { - | order-cleanup-interval = 30s - |} - |waves.miner.enable=no - """.stripMargin + """waves { + | matcher.order-cleanup-interval = 30s + | miner.enable=no + |}""".stripMargin ) val AssetQuantity: Long = 1000 @@ -129,7 +128,9 @@ object OrderExclusionTestSuite { val Waves: Long = 100000000L - val Configs: Seq[Config] = Seq(matcherConfig.withFallback(Default.head)) ++ - Random.shuffle(Default.tail.init).take(2).map(nonGeneratingPeersConfig.withFallback(_)) ++ - Random.shuffle(Default.tail.init).headOption + private val Configs: Seq[Config] = { + val notMatchingNodes = Random.shuffle(Default.init).take(3) + Seq(matcherConfig.withFallback(Default.last), notMatchingNodes.head) ++ + notMatchingNodes.tail.map(nonGeneratingPeersConfig.withFallback) + } } diff --git a/src/main/scala/com/wavesplatform/Application.scala b/src/main/scala/com/wavesplatform/Application.scala index c462d84da49..e2120ef35eb 100644 --- a/src/main/scala/com/wavesplatform/Application.scala +++ b/src/main/scala/com/wavesplatform/Application.scala @@ -218,7 +218,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con if (settings.restAPISettings.enable) { val apiRoutes = Seq( NodeApiRoute(settings.restAPISettings, blockchainUpdater, () => apiShutdown()), - BlocksApiRoute(settings.restAPISettings, blockchainUpdater, blockchainUpdater, allChannels, c => processCheckpoint(None, c)), + BlocksApiRoute(settings.restAPISettings, blockchainUpdater, allChannels, c => processCheckpoint(None, c)), TransactionsApiRoute(settings.restAPISettings, wallet, blockchainUpdater, utxStorage, allChannels, time), NxtConsensusApiRoute(settings.restAPISettings, blockchainUpdater, settings.blockchainSettings.functionalitySettings), WalletApiRoute(settings.restAPISettings, wallet), diff --git a/src/main/scala/com/wavesplatform/Explorer.scala b/src/main/scala/com/wavesplatform/Explorer.scala index e81927d2b34..2184b9c0783 100644 --- a/src/main/scala/com/wavesplatform/Explorer.scala +++ b/src/main/scala/com/wavesplatform/Explorer.scala @@ -1,9 +1,11 @@ package com.wavesplatform import java.io.File +import java.nio.ByteBuffer +import java.util import com.typesafe.config.ConfigFactory -import com.wavesplatform.database.LevelDBWriter +import com.wavesplatform.database.{Keys, LevelDBWriter} import com.wavesplatform.db.openDB import com.wavesplatform.settings.{WavesSettings, loadConfig} import com.wavesplatform.state.ByteStr @@ -12,9 +14,56 @@ import scorex.account.{Address, AddressScheme} import com.wavesplatform.utils.Base58 import scorex.utils.ScorexLogging +import scala.collection.JavaConverters._ import scala.util.Try +import scala.collection.JavaConverters._ object Explorer extends ScorexLogging { + case class Stats(entryCount: Long, totalKeySize: Long, totalValueSize: Long) + + private val keys = Array( + "version", + "height", + "score", + "block-at-height", + "height-of", + "waves-balance-history", + "waves-balance", + "assets-for-address", + "asset-balance-history", + "asset-balance", + "asset-info-history", + "asset-info", + "lease-balance-history", + "lease-balance", + "lease-status-history", + "lease-status", + "filled-volume-and-fee-history", + "filled-volume-and-fee", + "transaction-info", + "address-transaction-history", + "address-transaction-ids", + "changed-addresses", + "transaction-ids-at-height", + "address-id-of-alias", + "last-address-id", + "address-to-id", + "id-of-address", + "address-script-history", + "address-script", + "approved-features", + "activated-features", + "data-key-list", + "data-history", + "data", + "sponsorship-history", + "sponsorship", + "addresses-for-waves-seq-nr", + "addresses-for-waves", + "addresses-for-asset-seq-nr", + "addresses-for-asset" + ) + def main(args: Array[String]): Unit = { SLF4JBridgeHandler.removeHandlersForRootLogger() SLF4JBridgeHandler.install() @@ -41,18 +90,18 @@ object Explorer extends ScorexLogging { case "O" => val orderId = Base58.decode(args(2)).toOption.map(ByteStr.apply) if (orderId.isDefined) { - val kVolumeAndFee = LevelDBWriter.k.filledVolumeAndFee(blockchainHeight, orderId.get) + val kVolumeAndFee = Keys.filledVolumeAndFee(blockchainHeight, orderId.get) val bytes1 = db.get(kVolumeAndFee.keyBytes) val v = kVolumeAndFee.parse(bytes1) log.info(s"OrderId = ${Base58.encode(orderId.get.arr)}: Volume = ${v.volume}, Fee = ${v.fee}") - val kVolumeAndFeeHistory = LevelDBWriter.k.filledVolumeAndFeeHistory(orderId.get) + val kVolumeAndFeeHistory = Keys.filledVolumeAndFeeHistory(orderId.get) val bytes2 = db.get(kVolumeAndFeeHistory.keyBytes) val value2 = kVolumeAndFeeHistory.parse(bytes2) val value2Str = value2.mkString("[", ", ", "]") log.info(s"OrderId = ${Base58.encode(orderId.get.arr)}: History = $value2Str") value2.foreach { h => - val k = LevelDBWriter.k.filledVolumeAndFee(h, orderId.get) + val k = Keys.filledVolumeAndFee(h, orderId.get) val v = k.parse(db.get(k.keyBytes)) log.info(s"\t h = $h: Volume = ${v.volume}, Fee = ${v.fee}") } @@ -60,29 +109,47 @@ object Explorer extends ScorexLogging { case "A" => val address = Address.fromString(args(2)).right.get - val aid = LevelDBWriter.k.addressId(address) + val aid = Keys.addressId(address) val addressId = aid.parse(db.get(aid.keyBytes)).get log.info(s"Address id = $addressId") - val kwbh = LevelDBWriter.k.wavesBalanceHistory(addressId) + val kwbh = Keys.wavesBalanceHistory(addressId) val wbh = kwbh.parse(db.get(kwbh.keyBytes)) val balances = wbh.map { h => - val k = LevelDBWriter.k.wavesBalance(h, addressId) + val k = Keys.wavesBalance(h, addressId) h -> k.parse(db.get(k.keyBytes)) } balances.foreach(b => log.info(s"h = ${b._1}: balance = ${b._2}")) case "AC" => - val lastAddressId = LevelDBWriter.k.lastAddressId.parse(db.get(LevelDBWriter.k.lastAddressId.keyBytes)) + val lastAddressId = Keys.lastAddressId.parse(db.get(Keys.lastAddressId.keyBytes)) log.info(s"Last address id: $lastAddressId") + case "AD" => + val result = new util.HashMap[Address, java.lang.Integer]() + val lastAddressId = Keys.lastAddressId.parse(db.get(Keys.lastAddressId.keyBytes)) + for (id <- BigInt(1) to lastAddressId.getOrElse(BigInt(0))) { + val k = Keys.idToAddress(id) + val address = k.parse(db.get(k.keyBytes)) + result.compute(address, + (_, prev) => + prev match { + case null => 1 + case notNull => 1 + notNull + }) + } + + for ((k, v) <- result.asScala if v > 1) { + log.info(s"$k,$v") + } + case "T" => val address = Address.fromString(args(2)).right.get - val aid = LevelDBWriter.k.addressId(address) + val aid = Keys.addressId(address) val addressId = aid.parse(db.get(aid.keyBytes)).get log.info(s"Address id = $addressId") - val ktxidh = LevelDBWriter.k.addressTransactionIds(args(3).toInt, addressId) + val ktxidh = Keys.addressTransactionIds(args(3).toInt, addressId) for ((t, id) <- ktxidh.parse(db.get(ktxidh.keyBytes))) { log.info(s"$id of type $t") @@ -93,18 +160,42 @@ object Explorer extends ScorexLogging { val address = Address.fromString(args(2)).right.get val asset = ByteStr.decodeBase58(secondaryId).get - val ai = LevelDBWriter.k.addressId(address) + val ai = Keys.addressId(address) val addressId = ai.parse(db.get(ai.keyBytes)).get log.info(s"Address ID = $addressId") - val kabh = LevelDBWriter.k.assetBalanceHistory(addressId, asset) + val kabh = Keys.assetBalanceHistory(addressId, asset) val abh = kabh.parse(db.get(kabh.keyBytes)) val balances = abh.map { h => - val k = LevelDBWriter.k.assetBalance(h, addressId, asset) + val k = Keys.assetBalance(h, addressId, asset) h -> k.parse(db.get(k.keyBytes)) } balances.foreach(b => log.info(s"h = ${b._1}: balance = ${b._2}")) + + case "S" => + log.info("Collecting DB stats") + val iterator = db.iterator() + val result = new util.HashMap[Short, Stats] + iterator.seekToFirst() + while (iterator.hasNext) { + val entry = iterator.next() + val keyPrefix = ByteBuffer.wrap(entry.getKey).getShort + result.compute( + keyPrefix, + (_, maybePrev) => + maybePrev match { + case null => Stats(1, entry.getKey.length, entry.getValue.length) + case prev => Stats(prev.entryCount + 1, prev.totalKeySize + entry.getKey.length, prev.totalValueSize + entry.getValue.length) + } + ) + } + iterator.close() + + log.info("key-space,entry-count,total-key-size,total-value-size") + for ((prefix, stats) <- result.asScala) { + log.info(s"${keys(prefix)},${stats.entryCount},${stats.totalKeySize},${stats.totalValueSize}") + } } } finally db.close() } diff --git a/src/main/scala/com/wavesplatform/database/Keys.scala b/src/main/scala/com/wavesplatform/database/Keys.scala new file mode 100644 index 00000000000..aaf7887d1f7 --- /dev/null +++ b/src/main/scala/com/wavesplatform/database/Keys.scala @@ -0,0 +1,113 @@ +package com.wavesplatform.database + +import java.nio.ByteBuffer + +import com.google.common.primitives.{Ints, Longs, Shorts} +import com.wavesplatform.state._ +import scorex.account.{Address, Alias} +import scorex.block.{Block, BlockHeader} +import scorex.transaction.Transaction +import scorex.transaction.smart.script.{Script, ScriptReader} +import com.google.common.base.Charsets.UTF_8 + +object Keys { + private def h(prefix: Short, height: Int): Array[Byte] = + ByteBuffer.allocate(6).putShort(prefix).putInt(height).array() + + private def hBytes(prefix: Short, height: Int, bytes: Array[Byte]) = + ByteBuffer.allocate(6 + bytes.length).putShort(prefix).putInt(height).put(bytes).array() + + private def bytes(prefix: Short, bytes: Array[Byte]) = + ByteBuffer.allocate(2 + bytes.length).putShort(prefix).put(bytes).array() + + private def addr(prefix: Short, addressId: BigInt) = bytes(prefix, addressId.toByteArray) + + private def hash(prefix: Short, hashBytes: ByteStr) = bytes(prefix, hashBytes.arr) + + private def hAddr(prefix: Short, height: Int, addressId: BigInt): Array[Byte] = hBytes(prefix, height, addressId.toByteArray) + + private def historyKey(prefix: Short, b: Array[Byte]) = Key(bytes(prefix, b), readIntSeq, writeIntSeq) + + private def intKey(prefix: Short, default: Int = 0): Key[Int] = + Key[Int](Shorts.toByteArray(prefix), Option(_).fold(default)(Ints.fromByteArray), Ints.toByteArray) + + private def unsupported[A](message: String): A => Array[Byte] = _ => throw new UnsupportedOperationException(message) + + // actual key definition + + val version: Key[Int] = intKey(0, default = 1) + val height: Key[Int] = intKey(1) + def score(height: Int): Key[BigInt] = Key(h(2, height), Option(_).fold(BigInt(0))(BigInt(_)), _.toByteArray) + + private def blockAtHeight(height: Int) = h(3, height) + + def blockAt(height: Int): Key[Option[Block]] = Key.opt[Block](blockAtHeight(height), Block.parseBytes(_).get, _.bytes()) + def blockBytes(height: Int): Key[Option[Array[Byte]]] = Key.opt[Array[Byte]](blockAtHeight(height), identity, identity) + def blockHeader(height: Int): Key[Option[(BlockHeader, Int)]] = + Key.opt[(BlockHeader, Int)](blockAtHeight(height), b => (BlockHeader.parseBytes(b).get._1, b.length), unsupported("Can't write block headers")) // this dummy encoder is never used: we only store blocks, not block headers + + def heightOf(blockId: ByteStr): Key[Option[Int]] = Key.opt[Int](hash(4, blockId), Ints.fromByteArray, Ints.toByteArray) + + def wavesBalanceHistory(addressId: BigInt): Key[Seq[Int]] = historyKey(5, addressId.toByteArray) + def wavesBalance(height: Int, addressId: BigInt): Key[Long] = + Key(hAddr(6, height, addressId), Option(_).fold(0L)(Longs.fromByteArray), Longs.toByteArray) + + def assetList(addressId: BigInt): Key[Set[ByteStr]] = Key(addr(7, addressId), readTxIds(_).toSet, assets => writeTxIds(assets.toSeq)) + def assetBalanceHistory(addressId: BigInt, assetId: ByteStr): Key[Seq[Int]] = historyKey(8, addressId.toByteArray ++ assetId.arr) + def assetBalance(height: Int, addressId: BigInt, assetId: ByteStr): Key[Long] = + Key(hBytes(9, height, addressId.toByteArray ++ assetId.arr), Option(_).fold(0L)(Longs.fromByteArray), Longs.toByteArray) + + def assetInfoHistory(assetId: ByteStr): Key[Seq[Int]] = historyKey(10, assetId.arr) + def assetInfo(height: Int, assetId: ByteStr): Key[AssetInfo] = Key(hBytes(11, height, assetId.arr), readAssetInfo, writeAssetInfo) + + def leaseBalanceHistory(addressId: BigInt): Key[Seq[Int]] = historyKey(12, addressId.toByteArray) + def leaseBalance(height: Int, addressId: BigInt): Key[LeaseBalance] = + Key(hAddr(13, height, addressId), readLeaseBalance, writeLeaseBalance) + def leaseStatusHistory(leaseId: ByteStr): Key[Seq[Int]] = historyKey(14, leaseId.arr) + def leaseStatus(height: Int, leaseId: ByteStr): Key[Boolean] = + Key(hBytes(15, height, leaseId.arr), _(0) == 1, active => Array[Byte](if (active) 1 else 0)) + + def filledVolumeAndFeeHistory(orderId: ByteStr): Key[Seq[Int]] = historyKey(16, orderId.arr) + def filledVolumeAndFee(height: Int, orderId: ByteStr): Key[VolumeAndFee] = Key(hBytes(17, height, orderId.arr), readVolumeAndFee, writeVolumeAndFee) + + def transactionInfo(txId: ByteStr): Key[Option[(Int, Transaction)]] = Key.opt(hash(18, txId), readTransactionInfo, writeTransactionInfo) + def transactionHeight(txId: ByteStr): Key[Option[Int]] = + Key.opt(hash(18, txId), readTransactionHeight, unsupported("Can't write transaction height only")) + + def addressTransactionHistory(addressId: BigInt): Key[Seq[Int]] = historyKey(19, addressId.toByteArray) + def addressTransactionIds(height: Int, addressId: BigInt): Key[Seq[(Int, ByteStr)]] = + Key(hAddr(20, height, addressId), readTransactionIds, writeTransactionIds) + + def changedAddresses(height: Int): Key[Seq[BigInt]] = Key(h(21, height), readBigIntSeq, writeBigIntSeq) + + def transactionIdsAtHeight(height: Int): Key[Seq[ByteStr]] = Key(h(22, height), readTxIds, writeTxIds) + + def addressIdOfAlias(alias: Alias): Key[Option[BigInt]] = Key.opt(bytes(23, alias.bytes.arr), BigInt(_), _.toByteArray) + + val lastAddressId: Key[Option[BigInt]] = Key.opt(Array[Byte](0, 24), BigInt(_), _.toByteArray) + + def addressId(address: Address): Key[Option[BigInt]] = Key.opt(bytes(25, address.bytes.arr), BigInt(_), _.toByteArray) + def idToAddress(id: BigInt): Key[Address] = Key(bytes(26, id.toByteArray), Address.fromBytes(_).explicitGet(), _.bytes.arr) + + def addressScriptHistory(addressId: BigInt): Key[Seq[Int]] = historyKey(27, addressId.toByteArray) + def addressScript(height: Int, addressId: BigInt): Key[Option[Script]] = + Key.opt(hAddr(28, height, addressId), ScriptReader.fromBytes(_).explicitGet(), _.bytes().arr) + + def approvedFeatures: Key[Map[Short, Int]] = Key(Array[Byte](0, 29), readFeatureMap, writeFeatureMap) + def activatedFeatures: Key[Map[Short, Int]] = Key(Array[Byte](0, 30), readFeatureMap, writeFeatureMap) + + def dataKeyList(addressId: BigInt) = Key[Set[String]](addr(31, addressId), readStrings(_).toSet, keys => writeStrings(keys.toSeq)) + + def dataHistory(addressId: BigInt, key: String): Key[Seq[Int]] = historyKey(32, addressId.toByteArray ++ key.getBytes(UTF_8)) + def data(height: Int, addressId: BigInt, key: String): Key[Option[DataEntry[_]]] = + Key.opt(hBytes(33, height, addressId.toByteArray ++ key.getBytes(UTF_8)), DataEntry.parseValue(key, _, 0)._1, _.valueBytes) + + def sponsorshipHistory(assetId: ByteStr): Key[Seq[Int]] = historyKey(34, assetId.arr) + def sponsorship(height: Int, assetId: ByteStr): Key[SponsorshipValue] = Key(hBytes(35, height, assetId.arr), readSponsorship, writeSponsorship) + + val addressesForWavesSeqNr: Key[Int] = intKey(36) + def addressesForWaves(seqNr: Int): Key[Seq[BigInt]] = Key(h(37, seqNr), readBigIntSeq, writeBigIntSeq) + + def addressesForAssetSeqNr(assetId: ByteStr): Key[Int] = intKey(38) + def addressesForAsset(assetId: ByteStr, seqNr: Int): Key[Seq[BigInt]] = Key(hBytes(39, seqNr, assetId.arr), readBigIntSeq, writeBigIntSeq) +} diff --git a/src/main/scala/com/wavesplatform/database/LevelDBWriter.scala b/src/main/scala/com/wavesplatform/database/LevelDBWriter.scala index 5e4211ea300..ac859348929 100644 --- a/src/main/scala/com/wavesplatform/database/LevelDBWriter.scala +++ b/src/main/scala/com/wavesplatform/database/LevelDBWriter.scala @@ -1,11 +1,6 @@ package com.wavesplatform.database -import java.nio.ByteBuffer -import java.nio.charset.StandardCharsets - -import com.google.common.io.ByteStreams.{newDataInput, newDataOutput} -import com.google.common.primitives.{Ints, Longs, Shorts} -import com.wavesplatform.crypto +import com.google.common.cache.CacheBuilder import com.wavesplatform.settings.FunctionalitySettings import com.wavesplatform.state._ import com.wavesplatform.state.reader.LeaseDetails @@ -18,326 +13,28 @@ import scorex.transaction.assets._ import scorex.transaction.assets.exchange.ExchangeTransaction import scorex.transaction.lease.{LeaseCancelTransaction, LeaseTransaction} import scorex.transaction.smart.SetScriptTransaction -import scorex.transaction.smart.script.{Script, ScriptReader} +import scorex.transaction.smart.script.Script import scorex.transaction.transfer._ import scorex.utils.ScorexLogging import scala.annotation.tailrec -import scala.collection.immutable import scala.collection.mutable.ArrayBuffer +import scala.collection.{immutable, mutable} object LevelDBWriter { - object k { - import StandardCharsets.{UTF_8 => UTF8} - - private def h(prefix: Int, height: Int): Array[Byte] = { - val ndo = newDataOutput(6) - ndo.writeShort(prefix) - ndo.writeInt(height) - ndo.toByteArray - } - - private def byteKeyWithH(prefix: Int, height: Int, bytes: Array[Byte]) = { - val ndo = newDataOutput(6 + bytes.length) - ndo.writeShort(prefix) - ndo.writeInt(height) - ndo.write(bytes) - ndo.toByteArray - } - - private def byteKey(prefix: Int, bytes: Array[Byte]) = { - val ndo = newDataOutput(2 + bytes.length) - ndo.writeShort(prefix) - ndo.write(bytes) - ndo.toByteArray - } - - private def addr(prefix: Int, address: BigInt) = byteKey(prefix, address.toByteArray) - - private def hash(prefix: Int, hashBytes: ByteStr) = byteKey(prefix, hashBytes.arr) - - private def addressWithH(prefix: Int, height: Int, addressId: BigInt): Array[Byte] = { - val addressIdBytes = addressId.toByteArray - val ndo = newDataOutput(6 + addressIdBytes.length) - ndo.writeShort(prefix) - ndo.writeInt(height) - ndo.write(addressIdBytes) - ndo.toByteArray - } - - private def readStrings(data: Array[Byte]): Seq[String] = Option(data).fold(Seq.empty[String]) { d => - var i = 0 - val s = Seq.newBuilder[String] - - while (i < data.length) { - val len = Shorts.fromByteArray(data.drop(i)) - s += new String(data, i + 2, len, UTF8) - i += (2 + len) - } - s.result() - } - - private def writeStrings(strings: Seq[String]): Array[Byte] = { - val b = ByteBuffer.allocate(strings.map(_.getBytes(UTF8).length + 2).sum) - for (s <- strings) { - val bytes = s.getBytes(UTF8) - b.putShort(bytes.length.toShort).put(bytes) - } - b.array() - } - - private def readTxIds(data: Array[Byte]): Seq[ByteStr] = Option(data).fold(Seq.empty[ByteStr]) { d => - val b = ByteBuffer.wrap(d) - val ids = Seq.newBuilder[ByteStr] - - while (b.remaining() > 0) { - val buffer = b.get() match { - case crypto.DigestSize => new Array[Byte](crypto.DigestSize) - case crypto.SignatureLength => new Array[Byte](crypto.SignatureLength) - } - b.get(buffer) - ids += ByteStr(buffer) - } - - ids.result() - } - - private def writeTxIds(ids: Seq[ByteStr]): Array[Byte] = { - val b = ByteBuffer.allocate(ids.foldLeft(0) { case (prev, id) => prev + 1 + id.arr.length }) - for (id <- ids) { - b.put(id.arr.length match { - case crypto.DigestSize => crypto.DigestSize.toByte - case crypto.SignatureLength => crypto.SignatureLength.toByte - }) - .put(id.arr) - } - b.array() - } - - private def writeBigIntSeq(values: Seq[BigInt]) = { - val ndo = newDataOutput() - ndo.writeInt(values.size) - for (v <- values) { - ndo.writeBigInt(v) - } - ndo.toByteArray - } - - private def readBigIntSeq(data: Array[Byte]) = Option(data).fold(Seq.empty[BigInt]) { d => - val ndi = newDataInput(d) - val length = ndi.readInt() - for (_ <- 0 until length) yield ndi.readBigInt() - } - - private def historyKey(prefix: Int, bytes: Array[Byte]) = Key(byteKey(prefix, bytes), readIntSeq, writeIntSeq) - - val version = Key[Int](Array(0, 0), Option(_).fold(1)(Ints.fromByteArray), Ints.toByteArray) - val height = Key[Int](Array(0, 1), Option(_).fold(0)(Ints.fromByteArray), Ints.toByteArray) - - def score(height: Int) = Key[BigInt](h(2, height), Option(_).fold(BigInt(0))(BigInt(_)), _.toByteArray) - - private def blockAtHeight(height: Int) = h(3, height) - - def blockAt(height: Int) = Key.opt[Block](blockAtHeight(height), Block.parseBytes(_).get, _.bytes()) - - def blockBytes(height: Int) = Key.opt[Array[Byte]](blockAtHeight(height), identity, identity) - - def blockHeader(height: Int) = - Key.opt[(BlockHeader, Int)]( - blockAtHeight(height), - b => (BlockHeader.parseBytes(b).get._1, b.length), - _ => - throw new UnsupportedOperationException("Can't write block headers")) // this dummy encoder is never used: we only store blocks, not block headers - - def heightOf(blockId: ByteStr) = Key.opt[Int](hash(4, blockId), Ints.fromByteArray, Ints.toByteArray) - - def wavesBalanceHistory(addressId: BigInt): Key[Seq[Int]] = historyKey(5, addressId.toByteArray) - - def wavesBalance(height: Int, addressId: BigInt) = - Key[Long](addressWithH(6, height, addressId), Option(_).fold(0L)(Longs.fromByteArray), Longs.toByteArray) - - def assetList(addressId: BigInt) = Key[Set[ByteStr]](addr(7, addressId), readTxIds(_).toSet, assets => writeTxIds(assets.toSeq)) - - def assetBalanceHistory(addressId: BigInt, assetId: ByteStr) = historyKey(8, addressId.toByteArray ++ assetId.arr) - - def assetBalance(height: Int, addressId: BigInt, assetId: ByteStr) = - Key[Long](byteKeyWithH(9, height, addressId.toByteArray ++ assetId.arr), Option(_).fold(0L)(Longs.fromByteArray), Longs.toByteArray) - - private def readAssetInfo(data: Array[Byte]) = { - val ndi = newDataInput(data) - AssetInfo(ndi.readBoolean(), ndi.readBigInt(), ndi.readScriptOption()) - } - - private def writeAssetInfo(ai: AssetInfo): Array[Byte] = { - val ndo = newDataOutput() - ndo.writeBoolean(ai.isReissuable) - ndo.writeBigInt(ai.volume) - ndo.writeScriptOption(ai.script) - ndo.toByteArray - } - - def assetInfoHistory(assetId: ByteStr): Key[Seq[Int]] = historyKey(10, assetId.arr) - - def assetInfo(height: Int, assetId: ByteStr): Key[AssetInfo] = Key(byteKeyWithH(11, height, assetId.arr), readAssetInfo, writeAssetInfo) - - private def writeLeaseBalance(lb: LeaseBalance): Array[Byte] = { - val ndo = newDataOutput() - ndo.writeLong(lb.in) - ndo.writeLong(lb.out) - ndo.toByteArray - } - - private def readLeaseBalance(data: Array[Byte]) = Option(data).fold(LeaseBalance.empty) { d => - val ndi = newDataInput(d) - LeaseBalance(ndi.readLong(), ndi.readLong()) - } - - def leaseBalanceHistory(addressId: BigInt): Key[Seq[Int]] = historyKey(12, addressId.toByteArray) - - def leaseBalance(height: Int, addressId: BigInt): Key[LeaseBalance] = - Key(byteKeyWithH(13, height, addressId.toByteArray), readLeaseBalance, writeLeaseBalance) - - def leaseStatusHistory(leaseId: ByteStr): Key[Seq[Int]] = historyKey(14, leaseId.arr) - - def leaseStatus(height: Int, leaseId: ByteStr): Key[Boolean] = - Key(byteKeyWithH(15, height, leaseId.arr), _(0) == 1, active => Array[Byte](if (active) 1 else 0)) - - def filledVolumeAndFeeHistory(orderId: ByteStr): Key[Seq[Int]] = historyKey(16, orderId.arr) - - private def readVolumeAndFee(data: Array[Byte]) = Option(data).fold(VolumeAndFee.empty) { d => - val ndi = newDataInput(d) - VolumeAndFee(ndi.readLong(), ndi.readLong()) - } - - private def writeVolumeAndFee(vf: VolumeAndFee) = { - val ndo = newDataOutput() - ndo.writeLong(vf.volume) - ndo.writeLong(vf.fee) - ndo.toByteArray - } - - def filledVolumeAndFee(height: Int, orderId: ByteStr): Key[VolumeAndFee] = - Key(byteKeyWithH(17, height, orderId.arr), readVolumeAndFee, writeVolumeAndFee) - - private def readTransactionInfo(data: Array[Byte]) = - (Ints.fromByteArray(data), TransactionParsers.parseBytes(data.drop(4)).get) - - private def readTransactionHeight(data: Array[Byte]): Int = Ints.fromByteArray(data) - - private def writeTransactionInfo(txInfo: (Int, Transaction)) = { - val (h, tx) = txInfo - val txBytes = tx.bytes() - ByteBuffer.allocate(4 + txBytes.length).putInt(h).put(txBytes).array() - } - - def transactionInfo(txId: ByteStr): Key[Option[(Int, Transaction)]] = Key.opt(hash(18, txId), readTransactionInfo, writeTransactionInfo) - - def transactionHeight(txId: ByteStr): Key[Option[Int]] = - Key.opt(hash(18, txId), readTransactionHeight, _ => throw new UnsupportedOperationException("Can't write transaction height only")) - - def addressTransactionHistory(addressId: BigInt): Key[Seq[Int]] = historyKey(19, addressId.toByteArray) - - private def readTransactionIds(data: Array[Byte]) = Option(data).fold(Seq.empty[(Int, ByteStr)]) { d => - val b = ByteBuffer.wrap(d) - val ids = Seq.newBuilder[(Int, ByteStr)] - while (b.hasRemaining) { - ids += b.get.toInt -> { - val buf = new Array[Byte](b.get) - b.get(buf) - ByteStr(buf) - } - } - ids.result() - } - - private def writeTransactionIds(ids: Seq[(Int, ByteStr)]) = { - val size = ids.foldLeft(0) { case (prev, (_, id)) => prev + 2 + id.arr.length } - val buffer = ByteBuffer.allocate(size) - for ((typeId, id) <- ids) { - buffer.put(typeId.toByte).put(id.arr.length.toByte).put(id.arr) - } - buffer.array() - } - - def addressTransactionIds(height: Int, addressId: BigInt): Key[Seq[(Int, ByteStr)]] = - Key(byteKeyWithH(20, height, addressId.toByteArray), readTransactionIds, writeTransactionIds) - - def changedAddresses(height: Int): Key[Seq[BigInt]] = Key(h(21, height), readBigIntSeq, writeBigIntSeq) - - def transactionIdsAtHeight(height: Int): Key[Seq[ByteStr]] = Key(h(22, height), readTxIds, writeTxIds) - - def addressIdOfAlias(alias: Alias): Key[Option[BigInt]] = Key.opt(byteKey(23, alias.bytes.arr), BigInt(_), _.toByteArray) - - val lastAddressId: Key[Option[BigInt]] = Key.opt(Array[Byte](0, 24), BigInt(_), _.toByteArray) - - def addressId(address: Address): Key[Option[BigInt]] = Key.opt(byteKey(25, address.bytes.arr), BigInt(_), _.toByteArray) - - def idToAddress(id: BigInt): Key[Address] = Key(byteKey(26, id.toByteArray), Address.fromBytes(_).explicitGet(), _.bytes.arr) - - def addressScriptHistory(addressId: BigInt): Key[Seq[Int]] = historyKey(27, addressId.toByteArray) - - def addressScript(height: Int, addressId: BigInt): Key[Option[Script]] = - Key.opt(byteKeyWithH(28, height, addressId.toByteArray), ScriptReader.fromBytes(_).explicitGet(), _.bytes().arr) - - private def readFeatureMap(data: Array[Byte]): Map[Short, Int] = Option(data).fold(Map.empty[Short, Int]) { _ => - val b = ByteBuffer.wrap(data) - val features = Map.newBuilder[Short, Int] - while (b.hasRemaining) { - features += b.getShort -> b.getInt - } - - features.result() - } - - private def writeFeatureMap(features: Map[Short, Int]): Array[Byte] = { - val b = ByteBuffer.allocate(features.size * 6) - for ((featureId, height) <- features) - b.putShort(featureId).putInt(height) - - b.array() - } - - def approvedFeatures: Key[Map[Short, Int]] = Key(Array[Byte](0, 29), readFeatureMap, writeFeatureMap) - - def activatedFeatures: Key[Map[Short, Int]] = Key(Array[Byte](0, 30), readFeatureMap, writeFeatureMap) - - def dataKeyList(addressId: BigInt) = Key[Set[String]](addr(31, addressId), readStrings(_).toSet, keys => writeStrings(keys.toSeq)) - - def dataHistory(addressId: BigInt, key: String): Key[Seq[Int]] = historyKey(32, addressId.toByteArray ++ key.getBytes(UTF8)) - - def data(height: Int, addressId: BigInt, key: String): Key[Option[DataEntry[_]]] = - Key.opt(byteKeyWithH(33, height, addressId.toByteArray ++ key.getBytes(UTF8)), DataEntry.parseValue(key, _, 0)._1, _.valueBytes) - - private def readSponsorship(data: Array[Byte]) = { - val ndi = newDataInput(data) - SponsorshipValue(ndi.readLong()) - } - - private def writeSponsorship(ai: SponsorshipValue): Array[Byte] = { - val ndo = newDataOutput() - ndo.writeBigInt(ai.minFee) - ndo.toByteArray - } - - def sponsorshipHistory(assetId: ByteStr): Key[Seq[Int]] = historyKey(34, assetId.arr) - - def sponsorship(height: Int, assetId: ByteStr): Key[SponsorshipValue] = - Key(byteKeyWithH(35, height, assetId.arr), readSponsorship, writeSponsorship) - } - private def loadSponsorship(db: ReadOnlyDB, assetId: ByteStr) = { - db.get(k.sponsorshipHistory(assetId)).headOption.map(h => db.get(k.sponsorship(h, assetId))) + db.get(Keys.sponsorshipHistory(assetId)).headOption.map(h => db.get(Keys.sponsorship(h, assetId))) } private def loadAssetInfo(db: ReadOnlyDB, assetId: ByteStr) = { - db.get(k.assetInfoHistory(assetId)).headOption.map(h => db.get(k.assetInfo(h, assetId))) + db.get(Keys.assetInfoHistory(assetId)).headOption.map(h => db.get(Keys.assetInfo(h, assetId))) } private def loadLeaseStatus(db: ReadOnlyDB, leaseId: ByteStr) = - db.get(k.leaseStatusHistory(leaseId)).headOption.fold(false)(h => db.get(k.leaseStatus(h, leaseId))) + db.get(Keys.leaseStatusHistory(leaseId)).headOption.fold(false)(h => db.get(Keys.leaseStatus(h, leaseId))) private def resolveAlias(db: ReadOnlyDB, alias: Alias) = { - db.get(k.addressIdOfAlias(alias)).map(addressId => db.get(k.idToAddress(addressId))) + db.get(Keys.addressIdOfAlias(alias)).map(addressId => db.get(Keys.idToAddress(addressId))) } /** {{{ @@ -394,32 +91,26 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi finally rw.close() } - override protected def loadMaxAddressId(): BigInt = readOnly { db => - db.get(k.lastAddressId).getOrElse(BigInt(0)) - } + override protected def loadMaxAddressId(): BigInt = readOnly(db => db.get(Keys.lastAddressId).getOrElse(BigInt(0))) - override protected def loadAddressId(address: Address): Option[BigInt] = readOnly(db => db.get(k.addressId(address))) + override protected def loadAddressId(address: Address): Option[BigInt] = readOnly(db => db.get(Keys.addressId(address))) - override protected def loadHeight(): Int = readOnly(_.get(k.height)) + override protected def loadHeight(): Int = readOnly(_.get(Keys.height)) - override protected def loadScore(): BigInt = readOnly { db => - db.get(k.score(db.get(k.height))) - } + override protected def loadScore(): BigInt = readOnly(db => db.get(Keys.score(db.get(Keys.height)))) - override protected def loadLastBlock(): Option[Block] = readOnly { db => - db.get(k.blockAt(db.get(k.height))) - } + override protected def loadLastBlock(): Option[Block] = readOnly(db => db.get(Keys.blockAt(db.get(Keys.height)))) override protected def loadScript(address: Address): Option[Script] = readOnly { db => addressIdCache.get(address).fold[Option[Script]](None) { addressId => - loadFromHistory[Option[Script]](db, addressId, k.addressScriptHistory, k.addressScript).flatten + loadFromHistory[Option[Script]](db, addressId, Keys.addressScriptHistory, Keys.addressScript).flatten } } override def accountData(address: Address): AccountDataInfo = readOnly { db => val data = for { addressId <- addressIdCache.get(address).toSeq - key <- db.get(k.dataKeyList(addressId)) + key <- db.get(Keys.dataKeyList(addressId)) value <- accountData(address, key) } yield key -> value AccountDataInfo(data.toMap) @@ -427,15 +118,15 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi override def accountData(address: Address, key: String): Option[DataEntry[_]] = readOnly { db => addressIdCache.get(address).fold[Option[DataEntry[_]]](None) { addressId => - loadFromHistory[Option[DataEntry[_]]](db, addressId, k.dataHistory(_, key), k.data(_, _, key)).flatten + loadFromHistory[Option[DataEntry[_]]](db, addressId, Keys.dataHistory(_, key), Keys.data(_, _, key)).flatten } } override def balance(address: Address, mayBeAssetId: Option[AssetId]): Long = readOnly { db => addressIdCache.get(address).fold(0L) { addressId => mayBeAssetId match { - case Some(assetId) => loadFromHistory(db, addressId, k.assetBalanceHistory(_, assetId), k.assetBalance(_, _, assetId)).getOrElse(0L) - case None => loadFromHistory(db, addressId, k.wavesBalanceHistory, k.wavesBalance).getOrElse(0L) + case Some(assetId) => loadFromHistory(db, addressId, Keys.assetBalanceHistory(_, assetId), Keys.assetBalance(_, _, assetId)).getOrElse(0L) + case None => loadFromHistory(db, addressId, Keys.wavesBalanceHistory, Keys.wavesBalance).getOrElse(0L) } } } @@ -446,16 +137,16 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi } yield db.get(v(lastChange, addressId)) private def loadLposPortfolio(db: ReadOnlyDB, addressId: BigInt) = Portfolio( - loadFromHistory(db, addressId, k.wavesBalanceHistory, k.wavesBalance).getOrElse(0L), - loadFromHistory(db, addressId, k.leaseBalanceHistory, k.leaseBalance).getOrElse(LeaseBalance.empty), + loadFromHistory(db, addressId, Keys.wavesBalanceHistory, Keys.wavesBalance).getOrElse(0L), + loadFromHistory(db, addressId, Keys.leaseBalanceHistory, Keys.leaseBalance).getOrElse(LeaseBalance.empty), Map.empty ) private def loadPortfolio(db: ReadOnlyDB, addressId: BigInt) = loadLposPortfolio(db, addressId).copy( assets = (for { - assetId <- db.get(k.assetList(addressId)) - h <- db.get(k.assetBalanceHistory(addressId, assetId)).headOption - } yield assetId -> db.get(k.assetBalance(h, addressId, assetId))).toMap + assetId <- db.get(Keys.assetList(addressId)) + h <- db.get(Keys.assetBalanceHistory(addressId, assetId)).headOption + } yield assetId -> db.get(Keys.assetBalance(h, addressId, assetId))).toMap ) override protected def loadPortfolio(address: Address): Portfolio = readOnly { db => @@ -469,9 +160,9 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi readOnly(LevelDBWriter.loadSponsorship(_, assetId)) override protected def loadAssetDescription(assetId: ByteStr): Option[AssetDescription] = readOnly { db => - db.get(k.transactionInfo(assetId)) match { + db.get(Keys.transactionInfo(assetId)) match { case Some((_, i: IssueTransaction)) => - val ai = LevelDBWriter.loadAssetInfo(db, assetId).getOrElse(AssetInfo(false, 0, None)) + val ai = LevelDBWriter.loadAssetInfo(db, assetId).getOrElse(AssetInfo(isReissuable = false, 0, None)) val sponsorship = LevelDBWriter.loadSponsorship(db, assetId).fold(0L)(_.minFee) Some(AssetDescription(i.sender, i.name, i.description, i.decimals, ai.isReissuable, ai.volume, ai.script, sponsorship)) case _ => None @@ -479,15 +170,18 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi } override protected def loadVolumeAndFee(orderId: ByteStr): VolumeAndFee = readOnly { db => - db.get(k.filledVolumeAndFeeHistory(orderId)).headOption.fold(VolumeAndFee.empty)(h => db.get(k.filledVolumeAndFee(h, orderId))) + db.get(Keys.filledVolumeAndFeeHistory(orderId)).headOption.fold(VolumeAndFee.empty)(h => db.get(Keys.filledVolumeAndFee(h, orderId))) } - override protected def loadApprovedFeatures: Map[Short, Int] = readOnly(_.get(k.approvedFeatures)) + override protected def loadApprovedFeatures(): Map[Short, Int] = readOnly(_.get(Keys.approvedFeatures)) + + override protected def loadActivatedFeatures(): Map[Short, Int] = fs.preActivatedFeatures ++ readOnly(_.get(Keys.activatedFeatures)) - override protected def loadActivatedFeatures: Map[Short, Int] = fs.preActivatedFeatures ++ readOnly(_.get(k.activatedFeatures)) + private def updateHistory(rw: RW, key: Key[Seq[Int]], threshold: Int, kf: Int => Key[_]): Seq[Array[Byte]] = + updateHistory(rw, rw.get(key), key, threshold, kf) - private def updateHistory(rw: RW, key: Key[Seq[Int]], threshold: Int, kf: Int => Key[_]): Seq[Array[Byte]] = { - val (c1, c2) = rw.get(key).partition(_ > threshold) + private def updateHistory(rw: RW, history: Seq[Int], key: Key[Seq[Int]], threshold: Int, kf: Int => Key[_]): Seq[Array[Byte]] = { + val (c1, c2) = history.partition(_ > threshold) rw.put(key, (height +: c1) ++ c2.headOption) c2.drop(1).map(kf(_).keyBytes) } @@ -507,69 +201,96 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi sponsorship: Map[AssetId, Sponsorship]): Unit = readWrite { rw => val expiredKeys = new ArrayBuffer[Array[Byte]] - rw.put(k.height, height) - rw.put(k.blockAt(height), Some(block)) - rw.put(k.heightOf(block.uniqueId), Some(height)) - rw.put(k.lastAddressId, Some(loadMaxAddressId() + newAddresses.size)) - rw.put(k.score(height), rw.get(k.score(height - 1)) + block.blockScore()) + rw.put(Keys.height, height) + rw.put(Keys.blockAt(height), Some(block)) + rw.put(Keys.heightOf(block.uniqueId), Some(height)) + rw.put(Keys.lastAddressId, Some(loadMaxAddressId() + newAddresses.size)) + rw.put(Keys.score(height), rw.get(Keys.score(height - 1)) + block.blockScore()) for ((address, id) <- newAddresses) { - rw.put(k.addressId(address), Some(id)) - rw.put(k.idToAddress(id), address) + rw.put(Keys.addressId(address), Some(id)) + rw.put(Keys.idToAddress(id), address) } val threshold = height - 2000 val changedAddresses = Set.newBuilder[BigInt] + val newAddressesForWaves = ArrayBuffer.empty[BigInt] for ((addressId, balance) <- wavesBalances) { - rw.put(k.wavesBalance(height, addressId), balance) + val kwbh = Keys.wavesBalanceHistory(addressId) + val wbh = rw.get(kwbh) + if (wbh.isEmpty) { + newAddressesForWaves += addressId + } + rw.put(Keys.wavesBalance(height, addressId), balance) changedAddresses += addressId - expiredKeys ++= updateHistory(rw, k.wavesBalanceHistory(addressId), threshold, h => k.wavesBalance(h, addressId)) + expiredKeys ++= updateHistory(rw, wbh, kwbh, threshold, h => Keys.wavesBalance(h, addressId)) + } + + if (newAddressesForWaves.nonEmpty) { + val newSeqNr = rw.get(Keys.addressesForWavesSeqNr) + 1 + rw.put(Keys.addressesForWavesSeqNr, newSeqNr) + rw.put(Keys.addressesForWaves(newSeqNr), newAddressesForWaves) } for ((addressId, leaseBalance) <- leaseBalances) { - rw.put(k.leaseBalance(height, addressId), leaseBalance) + rw.put(Keys.leaseBalance(height, addressId), leaseBalance) changedAddresses += addressId - expiredKeys ++= updateHistory(rw, k.leaseBalanceHistory(addressId), threshold, k.leaseBalance(_, addressId)) + expiredKeys ++= updateHistory(rw, Keys.leaseBalanceHistory(addressId), threshold, Keys.leaseBalance(_, addressId)) } + val newAddressesForAsset = mutable.AnyRefMap.empty[ByteStr, Set[BigInt]] for ((addressId, assets) <- assetBalances) { - rw.put(k.assetList(addressId), rw.get(k.assetList(addressId)) ++ assets.keySet) + val prevAssets = rw.get(Keys.assetList(addressId)) + val newAssets = assets.keySet.diff(prevAssets) + for (assetId <- newAssets) { + newAddressesForAsset += assetId -> (newAddressesForAsset.getOrElse(assetId, Set.empty) + addressId) + } + rw.put(Keys.assetList(addressId), prevAssets ++ assets.keySet) changedAddresses += addressId for ((assetId, balance) <- assets) { - rw.put(k.assetBalance(height, addressId, assetId), balance) - expiredKeys ++= updateHistory(rw, k.assetBalanceHistory(addressId, assetId), threshold, k.assetBalance(_, addressId, assetId)) + rw.put(Keys.assetBalance(height, addressId, assetId), balance) + expiredKeys ++= updateHistory(rw, Keys.assetBalanceHistory(addressId, assetId), threshold, Keys.assetBalance(_, addressId, assetId)) } } - rw.put(k.changedAddresses(height), changedAddresses.result().toSeq) + for ((assetId, newAddressIds) <- newAddressesForAsset) { + val seqNrKey = Keys.addressesForAssetSeqNr(assetId) + val nextSeqNr = rw.get(seqNrKey) + 1 + val key = Keys.addressesForAsset(assetId, nextSeqNr) + + rw.put(seqNrKey, nextSeqNr) + rw.put(key, newAddressIds.toSeq) + } + + rw.put(Keys.changedAddresses(height), changedAddresses.result().toSeq) for ((orderId, volumeAndFee) <- filledQuantity) { - val kk = k.filledVolumeAndFee(height, orderId) + val kk = Keys.filledVolumeAndFee(height, orderId) rw.put(kk, volumeAndFee) - expiredKeys ++= updateHistory(rw, k.filledVolumeAndFeeHistory(orderId), threshold, k.filledVolumeAndFee(_, orderId)) + expiredKeys ++= updateHistory(rw, Keys.filledVolumeAndFeeHistory(orderId), threshold, Keys.filledVolumeAndFee(_, orderId)) } for ((assetId, assetInfo) <- reissuedAssets) { - rw.put(k.assetInfo(height, assetId), assetInfo) - expiredKeys ++= updateHistory(rw, k.assetInfoHistory(assetId), threshold, k.assetInfo(_, assetId)) + rw.put(Keys.assetInfo(height, assetId), assetInfo) + expiredKeys ++= updateHistory(rw, Keys.assetInfoHistory(assetId), threshold, Keys.assetInfo(_, assetId)) } for ((leaseId, state) <- leaseStates) { - rw.put(k.leaseStatus(height, leaseId), state) - expiredKeys ++= updateHistory(rw, k.leaseStatusHistory(leaseId), threshold, k.leaseStatus(_, leaseId)) + rw.put(Keys.leaseStatus(height, leaseId), state) + expiredKeys ++= updateHistory(rw, Keys.leaseStatusHistory(leaseId), threshold, Keys.leaseStatus(_, leaseId)) } for ((addressId, script) <- scripts) { - expiredKeys ++= updateHistory(rw, k.addressScriptHistory(addressId), threshold, k.addressScript(_, addressId)) - script.foreach(s => rw.put(k.addressScript(height, addressId), Some(s))) + expiredKeys ++= updateHistory(rw, Keys.addressScriptHistory(addressId), threshold, Keys.addressScript(_, addressId)) + script.foreach(s => rw.put(Keys.addressScript(height, addressId), Some(s))) } for ((addressId, addressData) <- data) { - rw.put(k.dataKeyList(addressId), rw.get(k.dataKeyList(addressId)) ++ addressData.data.keySet) + rw.put(Keys.dataKeyList(addressId), rw.get(Keys.dataKeyList(addressId)) ++ addressData.data.keySet) for ((key, value) <- addressData.data) { - rw.put(k.data(height, addressId, key), Some(value)) - expiredKeys ++= updateHistory(rw, k.dataHistory(addressId, key), threshold, k.data(_, addressId, key)) + rw.put(Keys.data(height, addressId, key), Some(value)) + expiredKeys ++= updateHistory(rw, Keys.dataHistory(addressId, key), threshold, Keys.data(_, addressId, key)) } } @@ -579,15 +300,15 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi } yield (addressId, (tx.builder.typeId.toInt, id))).groupBy(_._1).mapValues(_.map(_._2)) for ((addressId, txs) <- accountTransactions) { - rw.put(k.addressTransactionIds(height, addressId), txs) + rw.put(Keys.addressTransactionIds(height, addressId), txs) } for ((alias, addressId) <- aliases) { - rw.put(k.addressIdOfAlias(alias), Some(addressId)) + rw.put(Keys.addressIdOfAlias(alias), Some(addressId)) } for ((id, (tx, _)) <- transactions) { - rw.put(k.transactionInfo(id), Some((height, tx))) + rw.put(Keys.transactionInfo(id), Some((height, tx))) } val activationWindowSize = fs.activationWindowSize(height) @@ -598,57 +319,60 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi } if (newlyApprovedFeatures.nonEmpty) { - approvedFeaturesCache = newlyApprovedFeatures ++ rw.get(k.approvedFeatures) - rw.put(k.approvedFeatures, approvedFeaturesCache) + approvedFeaturesCache = newlyApprovedFeatures ++ rw.get(Keys.approvedFeatures) + rw.put(Keys.approvedFeatures, approvedFeaturesCache) - val featuresToSave = newlyApprovedFeatures.mapValues(_ + activationWindowSize) ++ rw.get(k.activatedFeatures) + val featuresToSave = newlyApprovedFeatures.mapValues(_ + activationWindowSize) ++ rw.get(Keys.activatedFeatures) activatedFeaturesCache = featuresToSave ++ fs.preActivatedFeatures - rw.put(k.activatedFeatures, featuresToSave) + rw.put(Keys.activatedFeatures, featuresToSave) } } for ((assetId, sp: SponsorshipValue) <- sponsorship) { - rw.put(k.sponsorship(height, assetId), sp) - expiredKeys ++= updateHistory(rw, k.sponsorshipHistory(assetId), threshold, k.sponsorship(_, assetId)) + rw.put(Keys.sponsorship(height, assetId), sp) + expiredKeys ++= updateHistory(rw, Keys.sponsorshipHistory(assetId), threshold, Keys.sponsorship(_, assetId)) } - rw.put(k.transactionIdsAtHeight(height), transactions.keys.toSeq) + rw.put(Keys.transactionIdsAtHeight(height), transactions.keys.toSeq) + expiredKeys.foreach(rw.delete) } override protected def doRollback(targetBlockId: ByteStr): Seq[Block] = { - readOnly(_.get(k.heightOf(targetBlockId))).fold(Seq.empty[Block]) { targetHeight => + readOnly(_.get(Keys.heightOf(targetBlockId))).fold(Seq.empty[Block]) { targetHeight => log.debug(s"Rolling back to block $targetBlockId at $targetHeight") val discardedBlocks = Seq.newBuilder[Block] for (currentHeight <- height until targetHeight by -1) readWrite { rw => - rw.put(k.height, currentHeight - 1) + rw.put(Keys.height, currentHeight - 1) - for (addressId <- rw.get(k.changedAddresses(currentHeight))) { - val address = rw.get(k.idToAddress(addressId)) + for (addressId <- rw.get(Keys.changedAddresses(currentHeight))) { + val address = rw.get(Keys.idToAddress(addressId)) - for (assetId <- rw.get(k.assetList(addressId))) { - rw.delete(k.assetBalance(currentHeight, addressId, assetId)) - rw.filterHistory(k.assetBalanceHistory(addressId, assetId), currentHeight) + for (assetId <- rw.get(Keys.assetList(addressId))) { + rw.delete(Keys.assetBalance(currentHeight, addressId, assetId)) + rw.filterHistory(Keys.assetBalanceHistory(addressId, assetId), currentHeight) } - rw.delete(k.wavesBalance(currentHeight, addressId)) - rw.filterHistory(k.wavesBalanceHistory(addressId), currentHeight) + rw.delete(Keys.wavesBalance(currentHeight, addressId)) + rw.filterHistory(Keys.wavesBalanceHistory(addressId), currentHeight) - rw.delete(k.leaseBalance(currentHeight, addressId)) - rw.filterHistory(k.leaseBalanceHistory(addressId), currentHeight) + rw.delete(Keys.leaseBalance(currentHeight, addressId)) + rw.filterHistory(Keys.leaseBalanceHistory(addressId), currentHeight) log.trace(s"Discarding portfolio for $address") portfolioCache.invalidate(address) + balanceAtHeightCache.invalidate((currentHeight, addressId)) + leaseBalanceAtHeightCache.invalidate((currentHeight, addressId)) } - val txIdsAtHeight = k.transactionIdsAtHeight(currentHeight) + val txIdsAtHeight = Keys.transactionIdsAtHeight(currentHeight) for (txId <- rw.get(txIdsAtHeight)) { forgetTransaction(txId) - val ktxId = k.transactionInfo(txId) + val ktxId = Keys.transactionInfo(txId) val Some((_, tx)) = rw.get(ktxId) rw.delete(ktxId) @@ -667,8 +391,8 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi val address = tx.sender.toAddress scriptCache.invalidate(address) addressIdCache.get(address).foreach { addressId => - rw.delete(k.addressScript(currentHeight, addressId)) - rw.filterHistory(k.addressScriptHistory(addressId), currentHeight) + rw.delete(Keys.addressScript(currentHeight, addressId)) + rw.filterHistory(Keys.addressScriptHistory(addressId), currentHeight) } case tx: DataTransaction => @@ -676,12 +400,12 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi addressIdCache.get(address).foreach { addressId => tx.data.foreach { e => log.trace(s"Discarding ${e.key} for $address at $currentHeight") - rw.delete(k.data(currentHeight, addressId, e.key)) - rw.filterHistory(k.dataHistory(addressId, e.key), currentHeight) + rw.delete(Keys.data(currentHeight, addressId, e.key)) + rw.filterHistory(Keys.dataHistory(addressId, e.key), currentHeight) } } - case tx: CreateAliasTransaction => rw.delete(k.addressIdOfAlias(tx.alias)) + case tx: CreateAliasTransaction => rw.delete(Keys.addressIdOfAlias(tx.alias)) case tx: ExchangeTransaction => rollbackOrderFill(rw, ByteStr(tx.buyOrder.id()), currentHeight) rollbackOrderFill(rw, ByteStr(tx.sellOrder.id()), currentHeight) @@ -691,13 +415,13 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi rw.delete(txIdsAtHeight) val discardedBlock = rw - .get(k.blockAt(currentHeight)) + .get(Keys.blockAt(currentHeight)) .getOrElse(throw new IllegalArgumentException(s"No block at height $currentHeight")) discardedBlocks += discardedBlock - rw.delete(k.blockAt(currentHeight)) - rw.delete(k.heightOf(discardedBlock.uniqueId)) + rw.delete(Keys.blockAt(currentHeight)) + rw.delete(Keys.heightOf(discardedBlock.uniqueId)) } log.debug(s"Rollback to block $targetBlockId at $targetHeight completed") @@ -707,41 +431,41 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi } private def rollbackAssetInfo(rw: RW, assetId: ByteStr, currentHeight: Int): Unit = { - rw.delete(k.assetInfo(currentHeight, assetId)) - rw.filterHistory(k.assetInfoHistory(assetId), currentHeight) + rw.delete(Keys.assetInfo(currentHeight, assetId)) + rw.filterHistory(Keys.assetInfoHistory(assetId), currentHeight) assetInfoCache.invalidate(assetId) assetDescriptionCache.invalidate(assetId) } private def rollbackOrderFill(rw: RW, orderId: ByteStr, currentHeight: Int): Unit = { - rw.delete(k.filledVolumeAndFee(currentHeight, orderId)) - rw.filterHistory(k.filledVolumeAndFeeHistory(orderId), currentHeight) + rw.delete(Keys.filledVolumeAndFee(currentHeight, orderId)) + rw.filterHistory(Keys.filledVolumeAndFeeHistory(orderId), currentHeight) volumeAndFeeCache.invalidate(orderId) } private def rollbackLeaseStatus(rw: RW, leaseId: ByteStr, currentHeight: Int): Unit = { - rw.delete(k.leaseStatus(currentHeight, leaseId)) - rw.filterHistory(k.leaseStatusHistory(leaseId), currentHeight) + rw.delete(Keys.leaseStatus(currentHeight, leaseId)) + rw.filterHistory(Keys.leaseStatusHistory(leaseId), currentHeight) } private def rollbackSponsorship(rw: RW, assetId: ByteStr, currentHeight: Int): Unit = { - rw.delete(k.sponsorship(currentHeight, assetId)) - rw.filterHistory(k.sponsorshipHistory(assetId), currentHeight) + rw.delete(Keys.sponsorship(currentHeight, assetId)) + rw.filterHistory(Keys.sponsorshipHistory(assetId), currentHeight) assetDescriptionCache.invalidate(assetId) sponsorshipCache.invalidate(assetId) } - override def transactionInfo(id: ByteStr): Option[(Int, Transaction)] = readOnly(db => db.get(k.transactionInfo(id))) + override def transactionInfo(id: ByteStr): Option[(Int, Transaction)] = readOnly(db => db.get(Keys.transactionInfo(id))) - override def transactionHeight(id: ByteStr): Option[Int] = readOnly(db => db.get(k.transactionHeight(id))) + override def transactionHeight(id: ByteStr): Option[Int] = readOnly(db => db.get(Keys.transactionHeight(id))) override def addressTransactions(address: Address, types: Set[Type], count: Int, from: Int): Seq[(Int, Transaction)] = readOnly { db => - db.get(k.addressId(address)).fold(Seq.empty[(Int, Transaction)]) { addressId => + db.get(Keys.addressId(address)).fold(Seq.empty[(Int, Transaction)]) { addressId => val txs = for { - h <- (db.get(k.height) to 1 by -1).view - (txType, txId) <- db.get(k.addressTransactionIds(h, addressId)) + h <- (db.get(Keys.height) to 1 by -1).view + (txType, txId) <- db.get(Keys.addressTransactionIds(h, addressId)) if types.isEmpty || types.contains(txType.toByte) - (_, tx) <- db.get(k.transactionInfo(txId)) + (_, tx) <- db.get(Keys.transactionInfo(txId)) } yield (h, tx) txs.slice(from, count).force @@ -751,31 +475,46 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi override def resolveAlias(a: Alias): Option[Address] = readOnly(db => LevelDBWriter.resolveAlias(db, a)) override def leaseDetails(leaseId: ByteStr): Option[LeaseDetails] = readOnly { db => - db.get(k.transactionInfo(leaseId)) match { + db.get(Keys.transactionInfo(leaseId)) match { case Some((h, lt: LeaseTransaction)) => Some(LeaseDetails(lt.sender, lt.recipient, h, lt.amount, loadLeaseStatus(db, leaseId))) case _ => None } } + // These two caches are used exclusively for balance snapshots. They are not used for portfolios, because there aren't + // as many miners, so snapshots will rarely be evicted due to overflows. + + private val balanceAtHeightCache = CacheBuilder + .newBuilder() + .maximumSize(100000) + .recordStats() + .build[(Int, BigInt), java.lang.Long]() + + private val leaseBalanceAtHeightCache = CacheBuilder + .newBuilder() + .maximumSize(100000) + .recordStats() + .build[(Int, BigInt), LeaseBalance]() + override def balanceSnapshots(address: Address, from: Int, to: Int): Seq[BalanceSnapshot] = readOnly { db => - db.get(k.addressId(address)).fold(Seq(BalanceSnapshot(1, 0, 0, 0))) { addressId => - val wbh = slice(db.get(k.wavesBalanceHistory(addressId)), from, to) - val lbh = slice(db.get(k.leaseBalanceHistory(addressId)), from, to) + db.get(Keys.addressId(address)).fold(Seq(BalanceSnapshot(1, 0, 0, 0))) { addressId => + val wbh = slice(db.get(Keys.wavesBalanceHistory(addressId)), from, to) + val lbh = slice(db.get(Keys.leaseBalanceHistory(addressId)), from, to) for { (wh, lh) <- merge(wbh, lbh) - wb = db.get(k.wavesBalance(wh, addressId)) - lb = db.get(k.leaseBalance(lh, addressId)) + wb = balanceAtHeightCache.get((wh, addressId), () => db.get(Keys.wavesBalance(wh, addressId))) + lb = leaseBalanceAtHeightCache.get((lh, addressId), () => db.get(Keys.leaseBalance(lh, addressId))) } yield BalanceSnapshot(wh.max(lh), wb, lb.in, lb.out) } } override def allActiveLeases: Set[LeaseTransaction] = readOnly { db => val txs = for { - h <- 1 to db.get(k.height) - id <- db.get(k.transactionIdsAtHeight(h)) + h <- 1 to db.get(Keys.height) + id <- db.get(Keys.transactionIdsAtHeight(h)) if loadLeaseStatus(db, id) - (_, tx) <- db.get(k.transactionInfo(id)) + (_, tx) <- db.get(Keys.transactionInfo(id)) } yield tx txs.collect { case lt: LeaseTransaction => lt }.toSet @@ -783,40 +522,40 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi override def collectLposPortfolios[A](pf: PartialFunction[(Address, Portfolio), A]) = readOnly { db => val b = Map.newBuilder[Address, A] - for (id <- BigInt(1) to db.get(k.lastAddressId).getOrElse(BigInt(0))) { - val address = db.get(k.idToAddress(id)) + for (id <- BigInt(1) to db.get(Keys.lastAddressId).getOrElse(BigInt(0))) { + val address = db.get(Keys.idToAddress(id)) pf.runWith(b += address -> _)(address -> loadLposPortfolio(db, id)) } b.result() } - override def scoreOf(blockId: ByteStr): Option[BigInt] = readOnly(db => db.get(k.heightOf(blockId)).map(h => db.get(k.score(h)))) + override def scoreOf(blockId: ByteStr): Option[BigInt] = readOnly(db => db.get(Keys.heightOf(blockId)).map(h => db.get(Keys.score(h)))) - override def blockHeaderAndSize(height: Int): Option[(BlockHeader, Int)] = readOnly(_.get(k.blockHeader(height))) + override def blockHeaderAndSize(height: Int): Option[(BlockHeader, Int)] = readOnly(_.get(Keys.blockHeader(height))) override def blockHeaderAndSize(blockId: ByteStr): Option[(BlockHeader, Int)] = - readOnly(db => db.get(k.heightOf(blockId)).flatMap(h => db.get(k.blockHeader(h)))) + readOnly(db => db.get(Keys.heightOf(blockId)).flatMap(h => db.get(Keys.blockHeader(h)))) - override def blockBytes(height: Int): Option[Array[Byte]] = readOnly(_.get(k.blockBytes(height))) + override def blockBytes(height: Int): Option[Array[Byte]] = readOnly(_.get(Keys.blockBytes(height))) override def blockBytes(blockId: ByteStr): Option[Array[Byte]] = - readOnly(db => db.get(k.heightOf(blockId)).flatMap(h => db.get(k.blockBytes(h)))) + readOnly(db => db.get(Keys.heightOf(blockId)).flatMap(h => db.get(Keys.blockBytes(h)))) - override def heightOf(blockId: ByteStr): Option[Int] = readOnly(_.get(k.heightOf(blockId))) + override def heightOf(blockId: ByteStr): Option[Int] = readOnly(_.get(Keys.heightOf(blockId))) override def lastBlockIds(howMany: Int): immutable.IndexedSeq[ByteStr] = readOnly { db => // since this is called from outside of the main blockchain updater thread, instead of using cached height, // explicitly read height from storage to make this operation atomic. - val currentHeight = db.get(k.height) + val currentHeight = db.get(Keys.height) (currentHeight until (currentHeight - howMany).max(0) by -1) - .map(h => db.get(k.blockHeader(h)).get._1.signerData.signature) + .map(h => db.get(Keys.blockHeader(h)).get._1.signerData.signature) } override def blockIdsAfter(parentSignature: ByteStr, howMany: Int): Option[Seq[ByteStr]] = readOnly { db => - db.get(k.heightOf(parentSignature)).map { parentHeight => + db.get(Keys.heightOf(parentSignature)).map { parentHeight => (parentHeight until (parentHeight + howMany)) .flatMap { h => - db.get(k.blockHeader(h)) + db.get(Keys.blockHeader(h)) } .map { b => b._1.signerData.signature @@ -825,37 +564,32 @@ class LevelDBWriter(writableDB: DB, fs: FunctionalitySettings) extends Caches wi } override def parent(block: Block, back: Int): Option[Block] = readOnly { db => - db.get(k.heightOf(block.reference)).flatMap(h => db.get(k.blockAt(h - back + 1))) + db.get(Keys.heightOf(block.reference)).flatMap(h => db.get(Keys.blockAt(h - back + 1))) } override def featureVotes(height: Int): Map[Short, Int] = readOnly { db => fs.activationWindow(height) - .flatMap(h => db.get(k.blockHeader(h)).fold(Seq.empty[Short])(_._1.featureVotes.toSeq)) + .flatMap(h => db.get(Keys.blockHeader(h)).fold(Seq.empty[Short])(_._1.featureVotes.toSeq)) .groupBy(identity) .mapValues(_.size) } - - private def distribution(height: Int, historyKey: BigInt => Key[Seq[Int]], balanceKey: (Int, BigInt) => Key[Long]): Map[Address, Long] = readOnly { - db => - val result = Map.newBuilder[Address, Long] - var lastAddressId = db.get(k.lastAddressId).getOrElse(BigInt(0)) - while (lastAddressId > 0) { - val abh = db.get(historyKey(lastAddressId)) - for (actualHeight <- abh.partition(_ > height)._2.headOption) { - val balance = db.get(balanceKey(actualHeight, lastAddressId)) - if (balance > 0) { - result += db.get(k.idToAddress(lastAddressId)) -> balance - } - } - - lastAddressId -= 1 - } - result.result() + override def assetDistribution(assetId: ByteStr): Map[Address, Long] = readOnly { db => + (for { + seqNr <- (1 to db.get(Keys.addressesForAssetSeqNr(assetId))).par + addressId <- db.get(Keys.addressesForAsset(assetId, seqNr)).par + balance <- loadFromHistory(db, addressId, Keys.assetBalanceHistory(_, assetId), Keys.assetBalance(_, _, assetId)) + if balance > 0 + } yield db.get(Keys.idToAddress(addressId)) -> balance).toMap.seq + } + + override def wavesDistribution(height: Int): Map[Address, Long] = readOnly { db => + (for { + seqNr <- (1 to db.get(Keys.addressesForWavesSeqNr)).par + addressId <- db.get(Keys.addressesForWaves(seqNr)).par + history = db.get(Keys.wavesBalanceHistory(addressId)) + actualHeight <- history.partition(_ > height)._2.headOption + balance = db.get(Keys.wavesBalance(actualHeight, addressId)) + if balance > 0 + } yield db.get(Keys.idToAddress(addressId)) -> balance).toMap.seq } - - override def assetDistribution(height: Int, assetId: ByteStr) = - distribution(height, k.assetBalanceHistory(_, assetId), k.assetBalance(_, _, assetId)) - - override def wavesDistribution(height: Int) = - distribution(height, k.wavesBalanceHistory, k.wavesBalance) } diff --git a/src/main/scala/com/wavesplatform/database/package.scala b/src/main/scala/com/wavesplatform/database/package.scala index ccd568ddfc8..e53f302885c 100644 --- a/src/main/scala/com/wavesplatform/database/package.scala +++ b/src/main/scala/com/wavesplatform/database/package.scala @@ -2,16 +2,20 @@ package com.wavesplatform import java.nio.ByteBuffer +import com.google.common.io.ByteStreams.{newDataInput, newDataOutput} import com.google.common.io.{ByteArrayDataInput, ByteArrayDataOutput} +import com.google.common.primitives.{Ints, Shorts} import com.wavesplatform.state._ import scorex.transaction.smart.script.{Script, ScriptReader} +import com.google.common.base.Charsets.UTF_8 +import scorex.transaction.{Transaction, TransactionParsers} package object database { implicit class ByteArrayDataOutputExt(val output: ByteArrayDataOutput) extends AnyVal { def writeBigInt(v: BigInt): Unit = { - val b = v.toByteArray - val len = b.length - output.writeByte(len) + val b = v.toByteArray + require(b.length <= Byte.MaxValue) + output.writeByte(b.length) output.write(b) } @@ -51,4 +55,168 @@ package object database { val in = ByteBuffer.wrap(data) Seq.fill(d.length / 4)(in.getInt) } + + def readTxIds(data: Array[Byte]): Seq[ByteStr] = Option(data).fold(Seq.empty[ByteStr]) { d => + val b = ByteBuffer.wrap(d) + val ids = Seq.newBuilder[ByteStr] + + while (b.remaining() > 0) { + val buffer = b.get() match { + case crypto.DigestSize => new Array[Byte](crypto.DigestSize) + case crypto.SignatureLength => new Array[Byte](crypto.SignatureLength) + } + b.get(buffer) + ids += ByteStr(buffer) + } + + ids.result() + } + + def writeTxIds(ids: Seq[ByteStr]): Array[Byte] = + ids + .foldLeft(ByteBuffer.allocate(ids.map(_.arr.length + 1).sum)) { + case (b, id) => + b.put(id.arr.length match { + case crypto.DigestSize => crypto.DigestSize.toByte + case crypto.SignatureLength => crypto.SignatureLength.toByte + }) + .put(id.arr) + } + .array() + + def readStrings(data: Array[Byte]): Seq[String] = Option(data).fold(Seq.empty[String]) { _ => + var i = 0 + val s = Seq.newBuilder[String] + + while (i < data.length) { + val len = Shorts.fromByteArray(data.drop(i)) + s += new String(data, i + 2, len, UTF_8) + i += (2 + len) + } + s.result() + } + + def writeStrings(strings: Seq[String]): Array[Byte] = + strings + .foldLeft(ByteBuffer.allocate(strings.map(_.getBytes(UTF_8).length + 2).sum)) { + case (b, s) => + val bytes = s.getBytes(UTF_8) + b.putShort(bytes.length.toShort).put(bytes) + } + .array() + + def writeBigIntSeq(values: Seq[BigInt]): Array[Byte] = { + require(values.length <= Short.MaxValue, s"BigInt sequence is too long") + val ndo = newDataOutput() + ndo.writeShort(values.size) + for (v <- values) { + ndo.writeBigInt(v) + } + ndo.toByteArray + } + + def readBigIntSeq(data: Array[Byte]): Seq[BigInt] = Option(data).fold(Seq.empty[BigInt]) { d => + val ndi = newDataInput(d) + val length = ndi.readShort() + for (_ <- 0 until length) yield ndi.readBigInt() + } + + def writeLeaseBalance(lb: LeaseBalance): Array[Byte] = { + val ndo = newDataOutput() + ndo.writeLong(lb.in) + ndo.writeLong(lb.out) + ndo.toByteArray + } + + def readLeaseBalance(data: Array[Byte]): LeaseBalance = Option(data).fold(LeaseBalance.empty) { d => + val ndi = newDataInput(d) + LeaseBalance(ndi.readLong(), ndi.readLong()) + } + + def readVolumeAndFee(data: Array[Byte]): VolumeAndFee = Option(data).fold(VolumeAndFee.empty) { d => + val ndi = newDataInput(d) + VolumeAndFee(ndi.readLong(), ndi.readLong()) + } + + def writeVolumeAndFee(vf: VolumeAndFee): Array[Byte] = { + val ndo = newDataOutput() + ndo.writeLong(vf.volume) + ndo.writeLong(vf.fee) + ndo.toByteArray + } + + def readTransactionInfo(data: Array[Byte]): (Int, Transaction) = + (Ints.fromByteArray(data), TransactionParsers.parseBytes(data.drop(4)).get) + + def readTransactionHeight(data: Array[Byte]): Int = Ints.fromByteArray(data) + + def writeTransactionInfo(txInfo: (Int, Transaction)) = { + val (h, tx) = txInfo + val txBytes = tx.bytes() + ByteBuffer.allocate(4 + txBytes.length).putInt(h).put(txBytes).array() + } + + def readTransactionIds(data: Array[Byte]): Seq[(Int, ByteStr)] = Option(data).fold(Seq.empty[(Int, ByteStr)]) { d => + val b = ByteBuffer.wrap(d) + val ids = Seq.newBuilder[(Int, ByteStr)] + while (b.hasRemaining) { + ids += b.get.toInt -> { + val buf = new Array[Byte](b.get) + b.get(buf) + ByteStr(buf) + } + } + ids.result() + } + + def writeTransactionIds(ids: Seq[(Int, ByteStr)]): Array[Byte] = { + val size = ids.foldLeft(0) { case (prev, (_, id)) => prev + 2 + id.arr.length } + val buffer = ByteBuffer.allocate(size) + for ((typeId, id) <- ids) { + buffer.put(typeId.toByte).put(id.arr.length.toByte).put(id.arr) + } + buffer.array() + } + + def readFeatureMap(data: Array[Byte]): Map[Short, Int] = Option(data).fold(Map.empty[Short, Int]) { _ => + val b = ByteBuffer.wrap(data) + val features = Map.newBuilder[Short, Int] + while (b.hasRemaining) { + features += b.getShort -> b.getInt + } + + features.result() + } + + def writeFeatureMap(features: Map[Short, Int]): Array[Byte] = { + val b = ByteBuffer.allocate(features.size * 6) + for ((featureId, height) <- features) + b.putShort(featureId).putInt(height) + + b.array() + } + + def readSponsorship(data: Array[Byte]): SponsorshipValue = { + val ndi = newDataInput(data) + SponsorshipValue(ndi.readLong()) + } + + def writeSponsorship(ai: SponsorshipValue): Array[Byte] = { + val ndo = newDataOutput() + ndo.writeBigInt(ai.minFee) + ndo.toByteArray + } + + def readAssetInfo(data: Array[Byte]): AssetInfo = { + val ndi = newDataInput(data) + AssetInfo(ndi.readBoolean(), ndi.readBigInt(), ndi.readScriptOption()) + } + + def writeAssetInfo(ai: AssetInfo): Array[Byte] = { + val ndo = newDataOutput() + ndo.writeBoolean(ai.isReissuable) + ndo.writeBigInt(ai.volume) + ndo.writeScriptOption(ai.script) + ndo.toByteArray + } } diff --git a/src/main/scala/com/wavesplatform/state/Blockchain.scala b/src/main/scala/com/wavesplatform/state/Blockchain.scala index 7f957b44ef7..f6fcd14c387 100644 --- a/src/main/scala/com/wavesplatform/state/Blockchain.scala +++ b/src/main/scala/com/wavesplatform/state/Blockchain.scala @@ -61,7 +61,7 @@ trait Blockchain { def balance(address: Address, mayBeAssetId: Option[AssetId]): Long - def assetDistribution(height: Int, assetId: ByteStr): Map[Address, Long] + def assetDistribution(assetId: ByteStr): Map[Address, Long] def wavesDistribution(height: Int): Map[Address, Long] // the following methods are used exclusively by patches diff --git a/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala b/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala index d40f3400fd6..e564cedc158 100644 --- a/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala +++ b/src/main/scala/com/wavesplatform/state/BlockchainUpdaterImpl.scala @@ -40,6 +40,8 @@ class BlockchainUpdaterImpl(blockchain: Blockchain, settings: WavesSettings, tim private val service = monix.execution.Scheduler.singleThread("last-block-info-publisher") private val internalLastBlockInfo = ConcurrentSubject.publish[LastBlockInfo](service) + override def isLastBlockId(id: ByteStr): Boolean = ngState.exists(_.contains(id)) || lastBlock.exists(_.uniqueId == id) + override val lastBlockInfo: Observable[LastBlockInfo] = internalLastBlockInfo.cache(1) lastBlockInfo.subscribe()(monix.execution.Scheduler.global) // Start caching @@ -494,14 +496,8 @@ class BlockchainUpdaterImpl(blockchain: Blockchain, settings: WavesSettings, tim } yield address -> f(address) } - override def assetDistribution(height: Int, assetId: AssetId): Map[Address, Long] = ngState.fold(blockchain.assetDistribution(height, assetId)) { - ng => - val innerDistribution = blockchain.assetDistribution(height, assetId) - if (height < this.height) innerDistribution - else { - innerDistribution ++ changedBalances(_.assets.getOrElse(assetId, 0L) != 0, portfolio(_).assets.getOrElse(assetId, 0L)) - } - } + override def assetDistribution(assetId: AssetId): Map[Address, Long] = + blockchain.assetDistribution(assetId) ++ changedBalances(_.assets.getOrElse(assetId, 0L) != 0, portfolio(_).assets.getOrElse(assetId, 0L)) override def wavesDistribution(height: Int): Map[Address, Long] = ngState.fold(blockchain.wavesDistribution(height)) { ng => val innerDistribution = blockchain.wavesDistribution(height) diff --git a/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala b/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala index 7ed8c8b4cc0..fa7bbfd1736 100644 --- a/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala +++ b/src/main/scala/com/wavesplatform/state/appender/BlockAppender.scala @@ -30,15 +30,13 @@ object BlockAppender extends ScorexLogging with Instrumented { Task { measureSuccessful( blockProcessingTimeStats, { - if (blockchainUpdater.contains(newBlock)) Right(None) - else - for { - _ <- Either.cond(blockchainUpdater.heightOf(newBlock.reference).exists(_ >= blockchainUpdater.height - 1), - (), - BlockAppendError("Irrelevant block", newBlock)) - _ = log.debug(s"Appending $newBlock") - maybeBaseHeight <- appendBlock(checkpoint, blockchainUpdater, utxStorage, time, settings)(newBlock) - } yield maybeBaseHeight map (_ => blockchainUpdater.score) + if (blockchainUpdater.isLastBlockId(newBlock.reference)) { + appendBlock(checkpoint, blockchainUpdater, utxStorage, time, settings)(newBlock).map(_ => Some(blockchainUpdater.score)) + } else if (blockchainUpdater.contains(newBlock.uniqueId)) { + Right(None) + } else { + Left(BlockAppendError("Block is not a child of the last block", newBlock)) + } } ) }.executeOn(scheduler) diff --git a/src/main/scala/com/wavesplatform/state/appender/package.scala b/src/main/scala/com/wavesplatform/state/appender/package.scala index ab5cea3cb91..82a7652ac29 100644 --- a/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -97,9 +97,9 @@ package object appender extends ScorexLogging { val fs = bcs.functionalitySettings val blockTime = block.timestamp val generator = block.signerData.generator + val height = blockchain.height val r: Either[ValidationError, Unit] = for { - height <- blockchain.heightOf(block.reference).toRight(GenericError(s"height: history does not contain parent ${block.reference}")) _ <- Either.cond( height > fs.blockVersion3AfterHeight || block.version == Block.GenesisBlockVersion diff --git a/src/main/scala/com/wavesplatform/state/package.scala b/src/main/scala/com/wavesplatform/state/package.scala index 715ee947af8..b1d463d2f6f 100644 --- a/src/main/scala/com/wavesplatform/state/package.scala +++ b/src/main/scala/com/wavesplatform/state/package.scala @@ -44,7 +44,7 @@ package object state { def blockAt(height: Int): Option[Block] = blockchain.blockBytes(height).flatMap(bb => Block.parseBytes(bb).toOption) def lastBlockHeaderAndSize: Option[(Block, Int)] = blockchain.lastBlock.map(b => (b, b.bytes().length)) - def lastBlockId: Option[ByteStr] = blockchain.lastBlockHeaderAndSize.map(_._1.signerData.signature) + def lastBlockId: Option[ByteStr] = blockchain.lastBlockHeaderAndSize.map(_._1.uniqueId) def lastBlockTimestamp: Option[Long] = blockchain.lastBlockHeaderAndSize.map(_._1.timestamp) def lastBlocks(howMany: Int): Seq[Block] = { diff --git a/src/main/scala/com/wavesplatform/state/reader/CompositeBlockchain.scala b/src/main/scala/com/wavesplatform/state/reader/CompositeBlockchain.scala index 8bd16e2c0a7..b3c3d335314 100644 --- a/src/main/scala/com/wavesplatform/state/reader/CompositeBlockchain.scala +++ b/src/main/scala/com/wavesplatform/state/reader/CompositeBlockchain.scala @@ -42,10 +42,8 @@ class CompositeBlockchain(inner: Blockchain, maybeDiff: => Option[Diff]) extends } case None => val sponsorship = diff.sponsorship.get(id).fold(0L) { - case SponsorshipValue(sponsorship) => - sponsorship - case SponsorshipNoInfo => - 0L + case SponsorshipValue(sp) => sp + case SponsorshipNoInfo => 0L } diff.transactions .get(id) @@ -156,13 +154,8 @@ class CompositeBlockchain(inner: Blockchain, maybeDiff: => Option[Diff]) extends if pred(p) } yield address -> f(address) - override def assetDistribution(height: Int, assetId: ByteStr): Map[Address, Long] = { - val innerDistribution = inner.assetDistribution(height, assetId) - if (height < this.height) innerDistribution - else { - innerDistribution ++ changedBalances(_.assets.getOrElse(assetId, 0L) != 0, portfolio(_).assets.getOrElse(assetId, 0L)) - } - } + override def assetDistribution(assetId: ByteStr): Map[Address, Long] = + inner.assetDistribution(assetId) ++ changedBalances(_.assets.getOrElse(assetId, 0L) != 0, portfolio(_).assets.getOrElse(assetId, 0L)) override def wavesDistribution(height: Int): Map[Address, Long] = { val innerDistribution = inner.wavesDistribution(height) diff --git a/src/main/scala/scorex/account/Address.scala b/src/main/scala/scorex/account/Address.scala index 1d88c2c4ecb..43f67b4db0e 100644 --- a/src/main/scala/scorex/account/Address.scala +++ b/src/main/scala/scorex/account/Address.scala @@ -1,9 +1,10 @@ package scorex.account +import java.nio.ByteBuffer + import com.wavesplatform.crypto import com.wavesplatform.state.ByteStr -import com.wavesplatform.utils.base58Length -import com.wavesplatform.utils.Base58 +import com.wavesplatform.utils.{Base58, base58Length} import scorex.transaction.ValidationError import scorex.transaction.ValidationError.InvalidAddress import scorex.utils.ScorexLogging @@ -22,7 +23,7 @@ object Address extends ScorexLogging { val AddressVersion: Byte = 1 val ChecksumLength = 4 val HashLength = 20 - val AddressLength = 1 + 1 + ChecksumLength + HashLength + val AddressLength = 1 + 1 + HashLength + ChecksumLength val AddressStringLength = base58Length(AddressLength) private def scheme = AddressScheme.current @@ -30,9 +31,9 @@ object Address extends ScorexLogging { private class AddressImpl(val bytes: ByteStr) extends Address def fromPublicKey(publicKey: Array[Byte], chainId: Byte = scheme.chainId): Address = { - val publicKeyHash = crypto.secureHash(publicKey).take(HashLength) - val withoutChecksum = AddressVersion +: chainId +: publicKeyHash - val bytes = withoutChecksum ++ calcCheckSum(withoutChecksum) + val publicKeyHash = crypto.secureHash(publicKey) + val withoutChecksum = ByteBuffer.allocate(1 + 1 + HashLength).put(AddressVersion).put(chainId).put(publicKeyHash, 0, HashLength).array() + val bytes = ByteBuffer.allocate(AddressLength).put(withoutChecksum).put(crypto.secureHash(withoutChecksum), 0, ChecksumLength).array() new AddressImpl(ByteStr(bytes)) } diff --git a/src/main/scala/scorex/api/http/BlocksApiRoute.scala b/src/main/scala/scorex/api/http/BlocksApiRoute.scala index 256d5adffac..492810eb75a 100644 --- a/src/main/scala/scorex/api/http/BlocksApiRoute.scala +++ b/src/main/scala/scorex/api/http/BlocksApiRoute.scala @@ -8,7 +8,7 @@ import com.wavesplatform.state.{Blockchain, ByteStr} import io.netty.channel.group.ChannelGroup import io.swagger.annotations._ import javax.ws.rs.Path -import monix.eval.{Coeval, Task} +import monix.eval.Task import monix.execution.Scheduler.Implicits.global import play.api.libs.json._ import scorex.block.BlockHeader @@ -21,7 +21,6 @@ import scala.util.Try @Api(value = "/blocks") case class BlocksApiRoute(settings: RestAPISettings, blockchain: Blockchain, - blockchainUpdater: BlockchainUpdater, allChannels: ChannelGroup, checkpointProc: Checkpoint => Task[Either[ValidationError, Option[BigInt]]]) extends ApiRoute { @@ -30,8 +29,6 @@ case class BlocksApiRoute(settings: RestAPISettings, val MaxBlocksPerRequest = 100 val rollbackExecutor = monix.execution.Scheduler.singleThread(name = "debug-rollback") - private val lastHeight: Coeval[Option[Int]] = lastObserved(blockchainUpdater.lastBlockInfo.map(_.height)) - override lazy val route = pathPrefix("blocks") { signature ~ first ~ last ~ lastHeaderOnly ~ at ~ atHeaderOnly ~ seq ~ seqHeaderOnly ~ height ~ heightEncoded ~ child ~ address ~ delay ~ checkpoint @@ -128,8 +125,7 @@ case class BlocksApiRoute(settings: RestAPISettings, @Path("/height") @ApiOperation(value = "Height", notes = "Get blockchain height", httpMethod = "GET") def height: Route = (path("height") & get) { - val x = lastHeight().getOrElse(0) - complete(Json.obj("height" -> x)) + complete(Json.obj("height" -> blockchain.height)) } @Path("/at/{height}") @@ -152,7 +148,7 @@ case class BlocksApiRoute(settings: RestAPISettings, (if (includeTransactions) { blockchain.blockAt(height).map(_.json()) } else { - blockchain.blockHeaderAndSize(height).map { case ((bh, s)) => BlockHeader.json(bh, s) } + blockchain.blockHeaderAndSize(height).map { case (bh, s) => BlockHeader.json(bh, s) } }) match { case Some(json) => complete(json + ("height" -> JsNumber(height))) case None => complete(Json.obj("status" -> "error", "details" -> "No block for this height")) @@ -187,7 +183,7 @@ case class BlocksApiRoute(settings: RestAPISettings, (if (includeTransactions) { blockchain.blockAt(height).map(_.json()) } else { - blockchain.blockHeaderAndSize(height).map { case ((bh, s)) => BlockHeader.json(bh, s) } + blockchain.blockHeaderAndSize(height).map { case (bh, s) => BlockHeader.json(bh, s) } }).map(_ + ("height" -> Json.toJson(height))) }) complete(blocks) diff --git a/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala b/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala index 8e44bc2484e..d09f47c62d9 100755 --- a/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala +++ b/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala @@ -60,7 +60,7 @@ case class AssetsApiRoute(settings: RestAPISettings, wallet: Wallet, utx: UtxPoo complete { Success(assetId).filter(_.length <= AssetIdStringLength).flatMap(Base58.decode) match { case Success(byteArray) => - Json.toJson(blockchain.assetDistribution(blockchain.height, ByteStr(byteArray)).map { case (a, b) => a.stringRepr -> b }) + Json.toJson(blockchain.assetDistribution(ByteStr(byteArray)).map { case (a, b) => a.stringRepr -> b }) case Failure(_) => ApiError.fromValidationError(scorex.transaction.ValidationError.GenericError("Must be base58-encoded assetId")) } } diff --git a/src/main/scala/scorex/transaction/BlockchainUpdater.scala b/src/main/scala/scorex/transaction/BlockchainUpdater.scala index c0e50049ab1..3eb51c61c8d 100644 --- a/src/main/scala/scorex/transaction/BlockchainUpdater.scala +++ b/src/main/scala/scorex/transaction/BlockchainUpdater.scala @@ -14,6 +14,8 @@ trait BlockchainUpdater { def lastBlockInfo: Observable[LastBlockInfo] + def isLastBlockId(id: ByteStr): Boolean + def shutdown(): Unit } diff --git a/src/main/scala/scorex/waves/http/DebugApiRoute.scala b/src/main/scala/scorex/waves/http/DebugApiRoute.scala index f46e030cafe..2bfcf36b194 100644 --- a/src/main/scala/scorex/waves/http/DebugApiRoute.scala +++ b/src/main/scala/scorex/waves/http/DebugApiRoute.scala @@ -125,8 +125,8 @@ case class DebugApiRoute(ws: WavesSettings, ) )) @ApiResponses(Array(new ApiResponse(code = 200, message = "Json portfolio"))) - def portfolios: Route = path("portfolios" / Segment) { (rawAddress) => - (get & withAuth & parameter('considerUnspent.as[Boolean])) { (considerUnspent) => + def portfolios: Route = path("portfolios" / Segment) { rawAddress => + (get & withAuth & parameter('considerUnspent.as[Boolean])) { considerUnspent => Address.fromString(rawAddress) match { case Left(_) => complete(InvalidAddress) case Right(address) => From 59af88c27227e190794ff9a77bfa2c7617b580e2 Mon Sep 17 00:00:00 2001 From: "Nastya.Urlapova" Date: Tue, 22 May 2018 12:41:10 +0300 Subject: [PATCH 26/52] NODE-575 Use fee from one place --- .../sync/AtomicSwapSmartContractSuite.scala | 6 --- .../it/sync/DataTransactionSuite.scala | 7 ---- .../sync/MassTransferTransactionSuite.scala | 42 ++++++++----------- .../com/wavesplatform/it/sync/package.scala | 24 +++++++++++ .../transactions/AliasTransactionSuite.scala | 11 ++--- 5 files changed, 46 insertions(+), 44 deletions(-) create mode 100644 it/src/test/scala/com/wavesplatform/it/sync/package.scala diff --git a/it/src/test/scala/com/wavesplatform/it/sync/AtomicSwapSmartContractSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/AtomicSwapSmartContractSuite.scala index c62ca0804e0..6abf059843c 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/AtomicSwapSmartContractSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/AtomicSwapSmartContractSuite.scala @@ -179,10 +179,4 @@ class AtomicSwapSmartContractSuite extends BaseTransactionSuite with CancelAfter nodes.waitForHeightAriseAndTxPresent(versionedTransferId) } - protected def calcDataFee(data: List[DataEntry[_]]): Long = { - val dataSize = data.map(_.toBytes.length).sum + 128 - if (dataSize > 1024) { - fee * (dataSize / 1024 + 1) - } else fee - } } diff --git a/it/src/test/scala/com/wavesplatform/it/sync/DataTransactionSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/DataTransactionSuite.scala index d9382bd4ad5..6d5d3cf3a2c 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/DataTransactionSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/DataTransactionSuite.scala @@ -276,11 +276,4 @@ class DataTransactionSuite extends BaseTransactionSuite { assertBadRequestAndResponse(sender.putData(firstAddress, tooManyEntriesData, calcDataFee(tooManyEntriesData)), message) nodes.waitForHeightArise() } - - private def calcDataFee(data: List[DataEntry[_]]): Long = { - val dataSize = data.map(_.toBytes.length).sum + 128 - if (dataSize > 1024) { - fee * (dataSize / 1024 + 1) - } else fee - } } diff --git a/it/src/test/scala/com/wavesplatform/it/sync/MassTransferTransactionSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/MassTransferTransactionSuite.scala index d08a8c45a02..2cccb8405aa 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/MassTransferTransactionSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/MassTransferTransactionSuite.scala @@ -17,17 +17,11 @@ import scala.util.Random class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfterFailure { - private val assetQuantity = 100.waves - private val transferAmount = 5.waves - private val leasingAmount = 5.waves - private val leasingFee = 0.003.waves - private val transferFee = notMiner.settings.feesSettings.fees(TransferTransactionV1.typeId)(0).fee - private val issueFee = 1.waves - private val massTransferFeePerTransfer = notMiner.settings.feesSettings.fees(MassTransferTransaction.typeId)(0).fee - - private def calcFee(numberOfRecipients: Int): Long = { - transferFee + massTransferFeePerTransfer * (numberOfRecipients + 1) - } + private val assetQuantity = 100.waves + private val transferAmount = 5.waves + private val leasingAmount = 5.waves + private val leasingFee = 0.003.waves + private val issueFee = 1.waves private def fakeSignature = Base58.encode(Array.fill(64)(Random.nextInt.toByte)) @@ -40,7 +34,7 @@ class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfter val assetId = sender.issue(firstAddress, "name", "description", assetQuantity, 8, reissuable = false, issueFee).id nodes.waitForHeightAriseAndTxPresent(assetId) - val massTransferTransactionFee = calcFee(transfers.size) + val massTransferTransactionFee = calcMassTransferFee(transfers.size) val transferId = sender.massTransfer(firstAddress, transfers, massTransferTransactionFee, Some(assetId)).id nodes.waitForHeightAriseAndTxPresent(transferId) @@ -57,7 +51,7 @@ class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfter val (balance3, eff3) = notMiner.accountBalances(thirdAddress) val transfers = List(Transfer(secondAddress, transferAmount), Transfer(thirdAddress, 2 * transferAmount)) - val massTransferTransactionFee = calcFee(transfers.size) + val massTransferTransactionFee = calcMassTransferFee(transfers.size) val transferId = sender.massTransfer(firstAddress, transfers, massTransferTransactionFee).id nodes.waitForHeightAriseAndTxPresent(transferId) @@ -73,7 +67,7 @@ class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfter val (balance2, eff2) = notMiner.accountBalances(secondAddress) val transfers = List(Transfer(secondAddress, balance1 / 2), Transfer(thirdAddress, balance1 / 2)) - assertBadRequestAndResponse(sender.massTransfer(firstAddress, transfers, calcFee(transfers.size)), "negative waves balance") + assertBadRequestAndResponse(sender.massTransfer(firstAddress, transfers, calcMassTransferFee(transfers.size)), "negative waves balance") nodes.waitForHeightArise() notMiner.assertBalances(firstAddress, balance1, eff1) @@ -86,7 +80,7 @@ class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfter val (balance2, eff2) = notMiner.accountBalances(secondAddress) val transfers = List(Transfer(secondAddress, transferAmount)) - assertBadRequestAndResponse(sender.massTransfer(firstAddress, transfers, transferFee), "Fee .* does not exceed minimal value") + assertBadRequestAndResponse(sender.massTransfer(firstAddress, transfers, fee), "Fee .* does not exceed minimal value") nodes.waitForHeightArise() notMiner.assertBalances(firstAddress, balance1, eff1) notMiner.assertBalances(secondAddress, balance2, eff2) @@ -95,12 +89,12 @@ class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfter test("can not make mass transfer without having enough of effective balance") { val (balance1, eff1) = notMiner.accountBalances(firstAddress) val (balance2, eff2) = notMiner.accountBalances(secondAddress) - val transfers = List(Transfer(secondAddress, balance1 - leasingFee - transferFee)) + val transfers = List(Transfer(secondAddress, balance1 - leasingFee - fee)) val leaseTxId = sender.lease(firstAddress, secondAddress, leasingAmount, leasingFee).id nodes.waitForHeightAriseAndTxPresent(leaseTxId) - assertBadRequestAndResponse(sender.massTransfer(firstAddress, transfers, calcFee(transfers.size)), "negative waves balance") + assertBadRequestAndResponse(sender.massTransfer(firstAddress, transfers, calcMassTransferFee(transfers.size)), "negative waves balance") nodes.waitForHeightArise() notMiner.assertBalances(firstAddress, balance1 - leasingFee, eff1 - leasingAmount - leasingFee) notMiner.assertBalances(secondAddress, balance2, eff2 + leasingAmount) @@ -111,7 +105,7 @@ class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfter def request(version: Byte = MassTransferTransaction.version, transfers: List[Transfer] = List(Transfer(secondAddress, transferAmount)), - fee: Long = calcFee(1), + fee: Long = calcMassTransferFee(1), timestamp: Long = System.currentTimeMillis, attachment: Array[Byte] = Array.emptyByteArray) = { val txEi = for { @@ -139,7 +133,7 @@ class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfter val (balance1, eff1) = notMiner.accountBalances(firstAddress) val invalidTransfers = Seq( (request(timestamp = System.currentTimeMillis + 1.day.toMillis), "Transaction .* is from far future"), - (request(transfers = List.fill(MaxTransferCount + 1)(Transfer(secondAddress, 1)), fee = calcFee(MaxTransferCount + 1)), + (request(transfers = List.fill(MaxTransferCount + 1)(Transfer(secondAddress, 1)), fee = calcMassTransferFee(MaxTransferCount + 1)), "Number of transfers is greater than 100"), (request(transfers = List(Transfer(secondAddress, -1))), "One of the transfers has negative amount"), (request(fee = 0), "insufficient fee"), @@ -158,7 +152,7 @@ class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfter test("huuuge transactions are allowed") { val (balance1, eff1) = notMiner.accountBalances(firstAddress) - val fee = calcFee(MaxTransferCount) + val fee = calcMassTransferFee(MaxTransferCount) val amount = (balance1 - fee) / MaxTransferCount val transfers = List.fill(MaxTransferCount)(Transfer(firstAddress, amount)) @@ -169,7 +163,7 @@ class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfter } test("transaction requires a proof") { - val fee = calcFee(2) + val fee = calcMassTransferFee(2) val transfers = Seq(Transfer(secondAddress, transferAmount), Transfer(thirdAddress, transferAmount)) val signedMassTransfer: JsObject = { val rs = sender.postJsonWithApiKey( @@ -206,16 +200,16 @@ class MassTransferTransactionSuite extends BaseTransactionSuite with CancelAfter val alias = "masstest_alias" val aliasFee = if (!sender.aliasByAddress(secondAddress).exists(_.endsWith(alias))) { - val aliasId = sender.createAlias(secondAddress, alias, transferFee).id + val aliasId = sender.createAlias(secondAddress, alias, fee).id nodes.waitForHeightAriseAndTxPresent(aliasId) - transferFee + fee } else 0 val aliasFull = sender.aliasByAddress(secondAddress).find(_.endsWith(alias)).get val transfers = List(Transfer(firstAddress, 0), Transfer(aliasFull, transferAmount)) - val massTransferTransactionFee = calcFee(transfers.size) + val massTransferTransactionFee = calcMassTransferFee(transfers.size) val transferId = sender.massTransfer(firstAddress, transfers, massTransferTransactionFee).id nodes.waitForHeightAriseAndTxPresent(transferId) diff --git a/it/src/test/scala/com/wavesplatform/it/sync/package.scala b/it/src/test/scala/com/wavesplatform/it/sync/package.scala new file mode 100644 index 00000000000..5f72b7062fb --- /dev/null +++ b/it/src/test/scala/com/wavesplatform/it/sync/package.scala @@ -0,0 +1,24 @@ +package com.wavesplatform.it + +import com.wavesplatform.state.DataEntry +import com.wavesplatform.it.util._ +import scorex.transaction.transfer.{MassTransferTransaction, TransferTransactionV1} + +package object sync { + val fee = 0.001.waves + val leasingFee = 0.001.waves + val massTransferFeePerTransfer = 0.0005.waves + val transferFee = fee + val transferAmount = 1.waves + + def calcDataFee(data: List[DataEntry[_]]): Long = { + val dataSize = data.map(_.toBytes.length).sum + 128 + if (dataSize > 1024) { + fee * (dataSize / 1024 + 1) + } else fee + } + + def calcMassTransferFee(numberOfRecipients: Int): Long = { + fee + massTransferFeePerTransfer * (numberOfRecipients + 1) + } +} diff --git a/it/src/test/scala/com/wavesplatform/it/sync/transactions/AliasTransactionSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/transactions/AliasTransactionSuite.scala index bb77f1dc68e..87f61bd5a7a 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/transactions/AliasTransactionSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/transactions/AliasTransactionSuite.scala @@ -4,15 +4,11 @@ import com.wavesplatform.it.api.SyncHttpApi._ import com.wavesplatform.it.transactions.BaseTransactionSuite import com.wavesplatform.it.util._ import org.scalatest.prop.TableDrivenPropertyChecks +import com.wavesplatform.it.sync._ import scala.util.Random class AliasTransactionSuite extends BaseTransactionSuite with TableDrivenPropertyChecks { - - private val transferFee = 1.waves - private val leasingFee = 0.001.waves - private val transferAmount = 1.waves - test("Able to send money to an alias") { val alias = randomAlias() val (balance1, eff1) = notMiner.accountBalances(firstAddress) @@ -113,8 +109,9 @@ class AliasTransactionSuite extends BaseTransactionSuite with TableDrivenPropert //previous test should not be commented to run this one test("Not able to create alias when insufficient funds") { - val alias = randomAlias() - assertBadRequestAndMessage(sender.createAlias(firstAddress, alias, transferFee), "State check failed. Reason: negative effective balance") + val balance = notMiner.accountBalances(firstAddress)._1 + val alias = randomAlias() + assertBadRequestAndMessage(sender.createAlias(firstAddress, alias, balance + transferFee), "State check failed. Reason: negative waves balance") } private def calcAliasFee(address: String, alias: String): Long = { From de3aa2a42a935e8f24a925877f942b480a589b54 Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Tue, 22 May 2018 12:55:50 +0300 Subject: [PATCH 27/52] Remove boolean comparation. Add string comparation. --- .../v1/evaluator/ctx/impl/PureContext.scala | 6 ++--- .../lang/v1/testing/ScriptGen.scala | 24 +------------------ 2 files changed, 4 insertions(+), 26 deletions(-) diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala index 8c90c5ab586..1b8f6f5f105 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala @@ -103,10 +103,10 @@ object PureContext { val eqString = createOp(EQ_OP, STRING, BOOLEAN)(_ == _) val ge = createOp(GE_OP, LONG, BOOLEAN)(_ >= _) val gt = createOp(GT_OP, LONG, BOOLEAN)(_ > _) - val bge = createOp(GE_OP, BOOLEAN, BOOLEAN)(_ >= _) - val bgt = createOp(GT_OP, BOOLEAN, BOOLEAN)(_ > _) + val sge = createOp(GE_OP, STRING, BOOLEAN)(_ >= _) + val sgt = createOp(GT_OP, STRING, BOOLEAN)(_ > _) - val operators: Seq[PredefFunction] = Seq(sumLong, subLong, sumString, sumByteVector, eqLong, eqByteVector, eqBool, eqString, ge, gt, bge, bgt, getElement, getListSize, uMinus, uNot) + val operators: Seq[PredefFunction] = Seq(sumLong, subLong, sumString, sumByteVector, eqLong, eqByteVector, eqBool, eqString, ge, gt, sge, sgt, getElement, getListSize, uMinus, uNot) lazy val instance = EvaluationContext.build(types = Seq.empty, diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index 97845cce778..bfc6fef865c 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -11,7 +11,7 @@ trait ScriptGen { def CONST_LONGgen: Gen[(EXPR, Long)] = Gen.choose(Long.MinValue, Long.MaxValue).map(v => (CONST_LONG(v), v)) def BOOLgen(gas: Int): Gen[(EXPR,Boolean)] = - if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), BGEgen(gas - 1), BGTgen(gas - 1), EQ_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1)) + if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1)) else Gen.const((TRUE, true)) def SUMgen(gas: Int): Gen[(EXPR, Long)] = @@ -30,28 +30,6 @@ trait ScriptGen { def INTGen(gas: Int): Gen[(EXPR, Long)] = if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), SUBgen(gas - 1), IF_INTgen(gas - 1), INTGen(gas-1).filter(v => (-BigInt(v._2)).isValidLong).map(e => (FUNCTION_CALL("-",List(e._1)), -e._2))) else CONST_LONGgen - def BGEgen(gas: Int): Gen[(EXPR, Boolean)] = - for { - dir <- Gen.oneOf(true, false) - (i1, v1) <- BOOLgen((gas - 2) / 2) - (i2, v2) <- BOOLgen((gas - 2) / 2) - } yield if(dir) { - (BINARY_OP(i1, GE_OP, i2), (v1 >= v2)) - } else { - (BINARY_OP(i2, LE_OP, i1), (v1 <= v2)) - } - - def BGTgen(gas: Int): Gen[(EXPR, Boolean)] = - for { - dir <- Gen.oneOf(true, false) - (i1, v1) <- BOOLgen((gas - 2) / 2) - (i2, v2) <- BOOLgen((gas - 2) / 2) - } yield if(dir) { - (BINARY_OP(i1, GT_OP, i2), (v1 > v2)) - } else { - (BINARY_OP(i2, LT_OP, i1), (v1 < v2)) - } - def GEgen(gas: Int): Gen[(EXPR, Boolean)] = for { dir <- Gen.oneOf(true, false) From 0906dd0def5da7e09afa0673bc46c8f9521abbbc Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Tue, 22 May 2018 13:10:43 +0300 Subject: [PATCH 28/52] `!=` operation --- .../lang/v1/evaluator/ctx/impl/PureContext.scala | 11 ++++++++++- .../lang/v1/parser/BinaryOperation.scala | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala index 1b8f6f5f105..e7fd80460ae 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala @@ -101,12 +101,21 @@ object PureContext { val eqByteVector = createOp(EQ_OP, BYTEVECTOR, BOOLEAN)(_ == _) val eqBool = createOp(EQ_OP, BOOLEAN, BOOLEAN)(_ == _) val eqString = createOp(EQ_OP, STRING, BOOLEAN)(_ == _) + val neLong = createOp(NE_OP, LONG, BOOLEAN)(_ != _) + val neByteVector = createOp(NE_OP, BYTEVECTOR, BOOLEAN)(_ != _) + val neBool = createOp(NE_OP, BOOLEAN, BOOLEAN)(_ != _) + val neString = createOp(NE_OP, STRING, BOOLEAN)(_ != _) val ge = createOp(GE_OP, LONG, BOOLEAN)(_ >= _) val gt = createOp(GT_OP, LONG, BOOLEAN)(_ > _) val sge = createOp(GE_OP, STRING, BOOLEAN)(_ >= _) val sgt = createOp(GT_OP, STRING, BOOLEAN)(_ > _) - val operators: Seq[PredefFunction] = Seq(sumLong, subLong, sumString, sumByteVector, eqLong, eqByteVector, eqBool, eqString, ge, gt, sge, sgt, getElement, getListSize, uMinus, uNot) + val operators: Seq[PredefFunction] = Seq(sumLong, subLong, sumString, sumByteVector, + eqLong, eqByteVector, eqBool, eqString, + neLong, neByteVector, neBool, neString, + ge, gt, sge, sgt, + getElement, getListSize, + uMinus, uNot) lazy val instance = EvaluationContext.build(types = Seq.empty, diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala index cd657b3928c..06591b88f3b 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala @@ -17,6 +17,7 @@ object BinaryOperation { OR_OP, AND_OP, EQ_OP, + NE_OP, GE_OP, GT_OP, LE_OP, @@ -36,6 +37,9 @@ object BinaryOperation { case object EQ_OP extends BinaryOperation { val func = "==" } + case object NE_OP extends BinaryOperation { + val func = "!=" + } case object GE_OP extends BinaryOperation { val func = ">=" } From c22ad1526f4442b07bd0282a5fba9ce8ec72e770 Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Tue, 22 May 2018 13:14:34 +0300 Subject: [PATCH 29/52] `!=` operanion test. --- .../com/wavesplatform/lang/v1/testing/ScriptGen.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index bfc6fef865c..02fc07f8996 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -11,7 +11,7 @@ trait ScriptGen { def CONST_LONGgen: Gen[(EXPR, Long)] = Gen.choose(Long.MinValue, Long.MaxValue).map(v => (CONST_LONG(v), v)) def BOOLgen(gas: Int): Gen[(EXPR,Boolean)] = - if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1)) + if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), NE_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1)) else Gen.const((TRUE, true)) def SUMgen(gas: Int): Gen[(EXPR, Long)] = @@ -58,6 +58,12 @@ trait ScriptGen { (i2, v2) <- INTGen((gas - 2) / 2) } yield (BINARY_OP(i1, EQ_OP, i2), (v1 == v2)) + def NE_INTgen(gas: Int): Gen[(EXPR, Boolean)] = + for { + (i1, v1) <- INTGen((gas - 2) / 2) + (i2, v2) <- INTGen((gas - 2) / 2) + } yield (BINARY_OP(i1, NE_OP, i2), (v1 != v2)) + def ANDgen(gas: Int): Gen[(EXPR, Boolean)] = for { (i1, v1) <- BOOLgen((gas - 2) / 2) From 76f450d907f87c5f37dc12e747aa1858c82d755b Mon Sep 17 00:00:00 2001 From: peterz Date: Tue, 22 May 2018 13:18:12 +0300 Subject: [PATCH 30/52] NODE-766 DataTransactionSpecification fails sometimes --- .../scorex/transaction/DataTransactionSpecification.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/scala/scorex/transaction/DataTransactionSpecification.scala b/src/test/scala/scorex/transaction/DataTransactionSpecification.scala index 8dfd3c34433..14fd87d62b4 100644 --- a/src/test/scala/scorex/transaction/DataTransactionSpecification.scala +++ b/src/test/scala/scorex/transaction/DataTransactionSpecification.scala @@ -41,7 +41,7 @@ class DataTransactionSpecification extends PropSpec with PropertyChecks with Mat } property("unknown type handing") { - val badTypeIdGen = Gen.choose[Byte](3, Byte.MaxValue) + val badTypeIdGen = Gen.choose[Int](DataEntry.Type.maxId + 1, Byte.MaxValue) forAll(dataTransactionGen, badTypeIdGen) { case (tx, badTypeId) => val bytes = tx.bytes() @@ -49,7 +49,7 @@ class DataTransactionSpecification extends PropSpec with PropertyChecks with Mat if (entryCount > 0) { val key1Length = Shorts.fromByteArray(bytes.drop(37)) val p = 39 + key1Length - bytes(p) = badTypeId + bytes(p) = badTypeId.toByte val parsed = DataTransaction.parseBytes(bytes) parsed.isFailure shouldBe true parsed.failed.get.getMessage shouldBe s"Unknown type $badTypeId" From 3917960be154216b3add398fe3b1b30dd364cca4 Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Tue, 22 May 2018 14:45:43 +0300 Subject: [PATCH 31/52] Make test more exact. --- .../wavesplatform/lang/IntegrationTest.scala | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala index ce2f59979e3..286e26f271d 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala @@ -90,21 +90,21 @@ class IntegrationTest extends PropSpec with PropertyChecks with ScriptGen with M } property("equals works on elements from Gens") { - List(CONST_LONGgen.map(_._1), SUMgen(50).map(_._1), INTGen(50).map(_._1)).foreach(gen => + List(CONST_LONGgen, SUMgen(50), INTGen(50)).foreach(gen => forAll(for { - expr <- gen - str <- toString(expr) - } yield str) { - case str => - eval[Boolean](s"$str == 0 || true") shouldBe Right(true) + (expr, res) <- gen + str <- toString(expr) + } yield (str, res)) { + case (str, res) => + eval[Long](str) shouldBe Right(res) }) forAll(for { - (expr, _) <- BOOLgen(50) - str <- toString(expr) - } yield str) { - case str => - eval[Boolean](s"$str || true") shouldBe Right(true) + (expr, res) <- BOOLgen(50) + str <- toString(expr) + } yield (str, res)) { + case (str, res) => + eval[Boolean](str) shouldBe Right(res) } } From ad0cfd9a264f058f6f34a79bf1b656448ab35963 Mon Sep 17 00:00:00 2001 From: Sergey Nazarov Date: Tue, 22 May 2018 17:05:24 +0300 Subject: [PATCH 32/52] IT tweaks --- .../com/wavesplatform/it/NodeConfigs.scala | 4 +- .../it/async/BlockSizeConstraintsSuite.scala | 87 +++++++++---------- .../it/async/MicroblocksGenerationSuite.scala | 45 +++++----- .../it/async/RollbackSpecSuite.scala | 5 +- 4 files changed, 68 insertions(+), 73 deletions(-) diff --git a/it/src/main/scala/com/wavesplatform/it/NodeConfigs.scala b/it/src/main/scala/com/wavesplatform/it/NodeConfigs.scala index 0ae7b79e5ad..2ff742f549f 100644 --- a/it/src/main/scala/com/wavesplatform/it/NodeConfigs.scala +++ b/it/src/main/scala/com/wavesplatform/it/NodeConfigs.scala @@ -10,7 +10,9 @@ object NodeConfigs { private val NonConflictingNodes = Set(1, 4, 6, 7) val Default: Seq[Config] = ConfigFactory.parseResources("nodes.conf").getConfigList("nodes").asScala - val NotMiner = Default.last + val Miners: Seq[Config] = Default.init + val NotMiner: Config = Default.last + def randomMiner: Config = Random.shuffle(Miners).head def newBuilder: Builder = Builder(Default, Default.size, Seq.empty) diff --git a/it/src/test/scala/com/wavesplatform/it/async/BlockSizeConstraintsSuite.scala b/it/src/test/scala/com/wavesplatform/it/async/BlockSizeConstraintsSuite.scala index 45c1b85983c..f9638dccf39 100644 --- a/it/src/test/scala/com/wavesplatform/it/async/BlockSizeConstraintsSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/async/BlockSizeConstraintsSuite.scala @@ -1,6 +1,6 @@ package com.wavesplatform.it.async -import com.typesafe.config.Config +import com.typesafe.config.{Config, ConfigFactory} import com.wavesplatform.it.api.AsyncHttpApi._ import com.wavesplatform.it.transactions.NodesFromDocker import com.wavesplatform.it.{NodeConfigs, TransferSending} @@ -11,52 +11,13 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ class BlockSizeConstraintsSuite extends FreeSpec with Matchers with TransferSending with NodesFromDocker { + import BlockSizeConstraintsSuite._ - private val maxTxs = 5000 // More, than 1mb of block - private val txsInMicroBlock = 500 - - override protected val nodeConfigs: Seq[Config] = NodeConfigs.newBuilder - .overrideBase( - _.raw( - s"""akka.http.server { - | parsing.max-content-length = 3737439 - | request-timeout = 60s - |} - | - |waves { - | network.enable-peers-exchange = no - | - | miner { - | quorum = 0 - | minimal-block-generation-offset = 60000ms - | micro-block-interval = 3s - | max-transactions-in-key-block = 0 - | max-transactions-in-micro-block = $txsInMicroBlock - | } - | - | blockchain.custom { - | functionality { - | feature-check-blocks-period = 1 - | blocks-for-feature-activation = 1 - | - | pre-activated-features { - | 2: 0 - | 3: 2 - | } - | } - | - | store-transactions-in-state = false - | } - | - | features.supported = [2, 3] - |}""".stripMargin - )) - .withDefault(1) - .build() + override protected val nodeConfigs: Seq[Config] = + Seq(ConfigOverrides.withFallback(NodeConfigs.randomMiner)) private val nodeAddresses = nodeConfigs.map(_.getString("address")).toSet - - private def miner = nodes.head + private val miner = nodes.head s"Block is limited by size after activation" in result( for { @@ -77,3 +38,41 @@ class BlockSizeConstraintsSuite extends FreeSpec with Matchers with TransferSend ) } + +object BlockSizeConstraintsSuite { + private val maxTxs = 5000 // More, than 1mb of block + private val txsInMicroBlock = 500 + private val ConfigOverrides = ConfigFactory.parseString(s"""akka.http.server { + | parsing.max-content-length = 3737439 + | request-timeout = 60s + |} + | + |waves { + | network.enable-peers-exchange = no + | + | miner { + | quorum = 0 + | minimal-block-generation-offset = 60000ms + | micro-block-interval = 3s + | max-transactions-in-key-block = 0 + | max-transactions-in-micro-block = $txsInMicroBlock + | } + | + | blockchain.custom { + | functionality { + | feature-check-blocks-period = 1 + | blocks-for-feature-activation = 1 + | + | pre-activated-features { + | 2: 0 + | 3: 2 + | } + | } + | + | store-transactions-in-state = false + | } + | + | features.supported = [2, 3] + |}""".stripMargin) + +} diff --git a/it/src/test/scala/com/wavesplatform/it/async/MicroblocksGenerationSuite.scala b/it/src/test/scala/com/wavesplatform/it/async/MicroblocksGenerationSuite.scala index c1161fa983a..dd6e72e27c2 100644 --- a/it/src/test/scala/com/wavesplatform/it/async/MicroblocksGenerationSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/async/MicroblocksGenerationSuite.scala @@ -1,6 +1,6 @@ package com.wavesplatform.it.async -import com.typesafe.config.Config +import com.typesafe.config.{Config, ConfigFactory} import com.wavesplatform.it.api.AsyncHttpApi._ import com.wavesplatform.it.transactions.NodesFromDocker import com.wavesplatform.it.{NodeConfigs, TransferSending} @@ -11,30 +11,10 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ class MicroblocksGenerationSuite extends FreeSpec with Matchers with TransferSending with NodesFromDocker { + import MicroblocksGenerationSuite._ - private val txsInMicroBlock = 200 - private val maxTxs = 2000 - - override protected val nodeConfigs: Seq[Config] = NodeConfigs.newBuilder - .overrideBase( - _.raw( - s"""waves { - | network.enable-peers-exchange = no - | - | miner { - | quorum = 0 - | minimal-block-generation-offset = 1m - | micro-block-interval = 3s - | max-transactions-in-key-block = 0 - | max-transactions-in-micro-block = $txsInMicroBlock - | } - | - | blockchain.custom.functionality.pre-activated-features.2 = 0 - | features.supported = [2] - |}""".stripMargin - )) - .withDefault(1) - .build() + override protected val nodeConfigs: Seq[Config] = + Seq(ConfigOverrides.withFallback(NodeConfigs.randomMiner)) private val nodeAddresses = nodeConfigs.map(_.getString("address")).toSet @@ -56,3 +36,20 @@ class MicroblocksGenerationSuite extends FreeSpec with Matchers with TransferSen ) } + +object MicroblocksGenerationSuite { + private val txsInMicroBlock = 200 + private val maxTxs = 2000 + private val ConfigOverrides = ConfigFactory.parseString(s"""waves { + | miner { + | quorum = 0 + | minimal-block-generation-offset = 1m + | micro-block-interval = 3s + | max-transactions-in-key-block = 0 + | max-transactions-in-micro-block = $txsInMicroBlock + | } + | + | blockchain.custom.functionality.pre-activated-features.2 = 0 + | features.supported = [2] + |}""".stripMargin) +} diff --git a/it/src/test/scala/com/wavesplatform/it/async/RollbackSpecSuite.scala b/it/src/test/scala/com/wavesplatform/it/async/RollbackSpecSuite.scala index 2a181d0f092..0585aa0eed5 100644 --- a/it/src/test/scala/com/wavesplatform/it/async/RollbackSpecSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/async/RollbackSpecSuite.scala @@ -25,10 +25,7 @@ class RollbackSpecSuite with WaitForHeight2 with NodesFromDocker { - override protected val nodeConfigs: Seq[Config] = Seq( - Random.shuffle(NodeConfigs.Default.init).head, - NodeConfigs.NotMiner - ) + override protected val nodeConfigs: Seq[Config] = Seq(NodeConfigs.randomMiner, NodeConfigs.NotMiner) private val nodeAddresses = nodeConfigs.map(_.getString("address")).toSet private val transactionsCount = 190 From 75a5127983d09f228809f8658ca9b6acafb2d0ba Mon Sep 17 00:00:00 2001 From: Alexandr M Date: Tue, 22 May 2018 17:56:28 +0300 Subject: [PATCH 33/52] NODE-129: FairPoSCalculator test added --- .../consensus/FairPoSCalculatorTest.scala | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala diff --git a/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala b/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala new file mode 100644 index 00000000000..eb64ba73890 --- /dev/null +++ b/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala @@ -0,0 +1,113 @@ +package com.wavesplatform.consensus + +import cats.data.NonEmptyList +import cats.implicits._ +import org.scalatest.{Matchers, PropSpec} +import scorex.account.PrivateKeyAccount + +import scala.util.Random + +class FairPoSCalculatorTest extends PropSpec with Matchers { + + val pos: PoSCalculator = FairPoSCalculator + + case class Block(height: Int, baseTarget: Long, miner: PrivateKeyAccount, timestamp: Long, delay: Long) + + def generationSignature: Array[Byte] = { + val arr = new Array[Byte](32) + Random.nextBytes(arr) + arr + } + + val balance = 50000000L * 100000000L + val blockDelaySeconds = 60 + val defaultBaseTarget = 100L + + property("Correct consensus parameters of blocks generated with FairPoS") { + + val miners = mkMiners + val first = Block(0, defaultBaseTarget, PrivateKeyAccount(generationSignature), System.currentTimeMillis(), 0) + + val chain = (1 to 100000 foldLeft NonEmptyList.of(first))((acc, _) => { + val gg = acc.tail.lift(1) + val blocks = miners.map(mineBlock(acc.head, gg, _)) + + val next = blocks.minBy(_.delay) + + next :: acc + }).reverse.tail + + val maxBT = chain.maxBy(_.baseTarget).baseTarget + val avgBT = chain.map(_.baseTarget).sum / chain.length + val minBT = chain.minBy(_.baseTarget).baseTarget + + val maxDelay = chain.tail.maxBy(_.delay).delay + val avgDelay = chain.tail.map(_.delay).sum / (chain.length - 1) + val minDelay = chain.tail.minBy(_.delay).delay + + println( + s""" + |BT: $minBT $avgBT $maxBT + |Delay: $minDelay $avgDelay $maxDelay + """.stripMargin + ) + + val perfomances = calcPerfomance(chain, miners) + + println(perfomances.toList.sortBy(_._1).mkString("\n")) + + assert(perfomances.forall(p => p._2 < 1.1 && p._2 > 0.9)) + assert(avgDelay < 80000 && avgDelay > 40000) + assert(avgBT < 200 && avgBT > 20) + } + + def mineBlock(prev: Block, grand: Option[Block], minerWithBalance: (PrivateKeyAccount, Long)): Block = { + val (miner, balance) = minerWithBalance + val gs = pos.generatorSignature(generationSignature, miner.publicKey) + val hit = pos.hit(gs) + val delay = pos.calculateDelay(hit, prev.baseTarget, balance) + val bt = pos.baseTarget( + blockDelaySeconds, + prev.height + 1, + prev.baseTarget, + prev.timestamp, + grand.map(_.timestamp), + prev.timestamp + delay + ) + + Block( + prev.height + 1, + bt, + miner, + prev.timestamp + delay, + delay + ) + } + + def calcPerfomance(chain: List[Block], miners: Map[PrivateKeyAccount, Long]): Map[Long, Double] = { + val balanceSum = miners.values.sum + val blocksCount = chain.length + + chain + .groupBy(_.miner) + .map(mbs => { + val (miner, blocks) = mbs + + val minerBalance = miners(miner) + val expectedBlocks = ((minerBalance.toDouble / balanceSum) * blocksCount).toLong + val perfomance = blocks.length.toDouble / expectedBlocks + + minerBalance -> perfomance + }) + } + + def mkMiners: Map[PrivateKeyAccount, Long] = + List( + PrivateKeyAccount(generationSignature) -> 200000000000000L, + PrivateKeyAccount(generationSignature) -> 500000000000000L, + PrivateKeyAccount(generationSignature) -> 1000000000000000L, + PrivateKeyAccount(generationSignature) -> 1500000000000000L, + PrivateKeyAccount(generationSignature) -> 2000000000000000L, + PrivateKeyAccount(generationSignature) -> 2500000000000000L + ).toMap +} From a8c8ea07515a1884f551a5fd5b6faec4f00484ce Mon Sep 17 00:00:00 2001 From: Alexandr M Date: Wed, 23 May 2018 12:17:30 +0300 Subject: [PATCH 34/52] NODE-129: Fix FairPoSTestSuite --- .../test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala index c8dc7e88c4a..0e76aca4e6f 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/FairPoSTestSuite.scala @@ -24,7 +24,7 @@ class FairPoSTestSuite extends FunSuite with CancelAfterFailure with NodesFromDo val heightAfterTransfer = nodes.head.height - nodes.head.waitForHeight(heightAfterTransfer + 20, 5.minutes) + nodes.head.waitForHeight(heightAfterTransfer + 20, 10.minutes) } } From 97f5f25193e7817fbdf12d920ee38c002b8250bd Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Wed, 23 May 2018 12:52:18 +0300 Subject: [PATCH 35/52] Make tests more stable. --- .../lang/v1/testing/ScriptGen.scala | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index 02fc07f8996..ea70c055e97 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -18,15 +18,23 @@ trait ScriptGen { for { (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - if((BigInt(v1) + BigInt(v2)).isValidLong) - } yield (BINARY_OP(i1, SUM_OP, i2), (v1 + v2)) + } yield + if((BigInt(v1) + BigInt(v2)).isValidLong) { + (BINARY_OP(i1, SUM_OP, i2), (v1 + v2)) + } else { + (BINARY_OP(i1, SUB_OP, i2), (v1 - v2)) + } def SUBgen(gas: Int): Gen[(EXPR, Long)] = for { (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - if((BigInt(v1) - BigInt(v2)).isValidLong) - } yield (BINARY_OP(i1, SUB_OP, i2), (v1 - v2)) + } yield + if((BigInt(v1) - BigInt(v2)).isValidLong) { + (BINARY_OP(i1, SUB_OP, i2), (v1 - v2)) + } else { + (BINARY_OP(i1, SUM_OP, i2), (v1 + v2)) + } def INTGen(gas: Int): Gen[(EXPR, Long)] = if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), SUBgen(gas - 1), IF_INTgen(gas - 1), INTGen(gas-1).filter(v => (-BigInt(v._2)).isValidLong).map(e => (FUNCTION_CALL("-",List(e._1)), -e._2))) else CONST_LONGgen From 74bd2203d7e77655a4702ffcc03d17cbc909c4a3 Mon Sep 17 00:00:00 2001 From: Alexandr M Date: Wed, 23 May 2018 13:57:42 +0300 Subject: [PATCH 36/52] NODE-129: Some functions moved from PosCalcuator trait to object,increased timeout in FairPoSSuite --- .../consensus/PoSCalculator.scala | 62 ++++++++++--------- .../wavesplatform/consensus/PoSSelector.scala | 32 +++------- .../consensus/FairPoSCalculatorTest.scala | 16 ++--- 3 files changed, 50 insertions(+), 60 deletions(-) diff --git a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala index 24312232d0b..4b34753a74a 100644 --- a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala +++ b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala @@ -3,36 +3,37 @@ package com.wavesplatform.consensus import com.wavesplatform.crypto trait PoSCalculator { - protected val HitSize: Int = 8 - protected val MinBaseTarget: Long = 9 + def calculateBaseTarget(targetBlockDelaySeconds: Long, + prevHeight: Int, + prevBaseTarget: Long, + parentTimestamp: Long, + maybeGreatGrandParentTimestamp: Option[Long], + timestamp: Long): Long - def generatorSignature(signature: Array[Byte], publicKey: Array[Byte]): Array[Byte] = { + def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long +} + +object PoSCalculator { + private[consensus] val HitSize: Int = 8 + private[consensus] val MinBaseTarget: Long = 9 + + private[consensus] def generatorSignature(signature: Array[Byte], publicKey: Array[Byte]): Array[Byte] = { val s = new Array[Byte](crypto.DigestSize * 2) System.arraycopy(signature, 0, s, 0, crypto.DigestSize) System.arraycopy(publicKey, 0, s, crypto.DigestSize, crypto.DigestSize) crypto.fastHash(s) } - def baseTarget(targetBlockDelaySeconds: Long, - prevHeight: Int, - prevBaseTarget: Long, - parentTimestamp: Long, - maybeGreatGrandParentTimestamp: Option[Long], - timestamp: Long): Long - - def hit(generatorSignature: Array[Byte]): BigInt = BigInt(1, generatorSignature.take(HitSize).reverse) + private[consensus] def hit(generatorSignature: Array[Byte]): BigInt = BigInt(1, generatorSignature.take(HitSize).reverse) - def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long - - def normalize(value: Long, targetBlockDelaySeconds: Long): Double = + private[consensus] def normalize(value: Long, targetBlockDelaySeconds: Long): Double = value * targetBlockDelaySeconds / (60: Double) - def normalizeBaseTarget(baseTarget: Long, targetBlockDelaySeconds: Long): Long = { + private[consensus] def normalizeBaseTarget(baseTarget: Long, targetBlockDelaySeconds: Long): Long = { baseTarget .max(MinBaseTarget) .min(Long.MaxValue / targetBlockDelaySeconds) } - } object NxtPoSCalculator extends PoSCalculator { @@ -41,12 +42,14 @@ object NxtPoSCalculator extends PoSCalculator { protected val BaseTargetGamma = 64 protected val MeanCalculationDepth = 3 - def baseTarget(targetBlockDelaySeconds: Long, - prevHeight: Int, - prevBaseTarget: Long, - parentTimestamp: Long, - maybeGreatGrandParentTimestamp: Option[Long], - timestamp: Long): Long = { + import PoSCalculator._ + + def calculateBaseTarget(targetBlockDelaySeconds: Long, + prevHeight: Int, + prevBaseTarget: Long, + parentTimestamp: Long, + maybeGreatGrandParentTimestamp: Option[Long], + timestamp: Long): Long = { if (prevHeight % 2 == 0) { val meanBlockDelay = maybeGreatGrandParentTimestamp.fold(timestamp - parentTimestamp)(ts => (timestamp - ts) / MeanCalculationDepth) / 1000 @@ -72,6 +75,9 @@ object NxtPoSCalculator extends PoSCalculator { } object FairPoSCalculator extends PoSCalculator { + + import PoSCalculator._ + private val MaxSignature: Array[Byte] = Array.fill[Byte](HitSize)(-1) private val MaxHit: BigDecimal = BigDecimal(BigInt(1, MaxSignature)) private val C1 = 70000 @@ -84,12 +90,12 @@ object FairPoSCalculator extends PoSCalculator { a.toLong } - def baseTarget(targetBlockDelaySeconds: Long, - prevHeight: Int, - prevBaseTarget: Long, - parentTimestamp: Long, - maybeGreatGrandParentTimestamp: Option[Long], - timestamp: Long): Long = { + def calculateBaseTarget(targetBlockDelaySeconds: Long, + prevHeight: Int, + prevBaseTarget: Long, + parentTimestamp: Long, + maybeGreatGrandParentTimestamp: Option[Long], + timestamp: Long): Long = { val maxDelay = normalize(90, targetBlockDelaySeconds) val minDelay = normalize(30, targetBlockDelaySeconds) diff --git a/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala b/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala index 80205218c95..9c2b55904d9 100644 --- a/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala +++ b/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala @@ -14,6 +14,8 @@ import scala.concurrent.duration.FiniteDuration class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { + import PoSCalculator._ + protected def pos(height: Int): PoSCalculator = if (fairPosActivated(height)) FairPoSCalculator else NxtPoSCalculator @@ -25,10 +27,10 @@ class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { refBlockTS: Long, greatGrandParentTS: Option[Long], currentTime: Long): Either[ValidationError, NxtLikeConsensusBlockData] = { - val bt = baseTarget(targetBlockDelay.toSeconds, height, refBlockBT, refBlockTS, greatGrandParentTS, currentTime) + val bt = pos(height).calculateBaseTarget(targetBlockDelay.toSeconds, height, refBlockBT, refBlockTS, greatGrandParentTS, currentTime) blockchain.lastBlock .map(_.consensusData.generationSignature.arr) - .map(gs => NxtLikeConsensusBlockData(bt, ByteStr(pos(height).generatorSignature(gs, accountPublicKey)))) + .map(gs => NxtLikeConsensusBlockData(bt, ByteStr(generatorSignature(gs, accountPublicKey)))) .toRight(GenericError("No blocks in blockchain")) } @@ -49,7 +51,7 @@ class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { def validateGeneratorSignature(height: Int, block: Block): Either[ValidationError, Unit] = { blockchain.lastBlock - .map(b => pos(height).generatorSignature(b.consensusData.generationSignature.arr, block.signerData.generator.publicKey)) + .map(b => generatorSignature(b.consensusData.generationSignature.arr, block.signerData.generator.publicKey)) .toRight(GenericError("No blocks in blockchain T.T")) .ensure(GenericError("Generation signatures doesnot match"))(_ sameElements block.consensusData.generationSignature.arr) .map(_ => ()) @@ -59,7 +61,7 @@ class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { val blockBT = block.consensusData.baseTarget val blockTS = block.timestamp - val expectedBT = baseTarget( + val expectedBT = pos(height).calculateBaseTarget( settings.genesisSettings.averageBlockDelay.toSeconds, height, parent.consensusData.baseTarget, @@ -81,28 +83,10 @@ class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { else blockchain.lastBlock blockForHit.map(b => { - val p = pos(height) - val gs = p.generatorSignature(b.consensusData.generationSignature.arr, accountPublicKey) - p.hit(gs) + val genSig = b.consensusData.generationSignature.arr + hit(generatorSignature(genSig, accountPublicKey)) }) } - def baseTarget(targetBlockDelaySeconds: Long, - prevHeight: Int, - prevBaseTarget: Long, - parentTimestamp: Long, - maybeGreatGrandParentTimestamp: Option[Long], - timestamp: Long): Long = { - pos(prevHeight) - .baseTarget( - targetBlockDelaySeconds, - prevHeight, - prevBaseTarget, - parentTimestamp, - maybeGreatGrandParentTimestamp, - timestamp - ) - } - private def fairPosActivated(height: Int): Boolean = blockchain.activatedFeaturesAt(height).contains(BlockchainFeatures.FairPoS.id) } diff --git a/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala b/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala index eb64ba73890..49e024f44c9 100644 --- a/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala +++ b/src/test/scala/com/wavesplatform/consensus/FairPoSCalculatorTest.scala @@ -9,6 +9,8 @@ import scala.util.Random class FairPoSCalculatorTest extends PropSpec with Matchers { + import PoSCalculator._ + val pos: PoSCalculator = FairPoSCalculator case class Block(height: Int, baseTarget: Long, miner: PrivateKeyAccount, timestamp: Long, delay: Long) @@ -52,21 +54,19 @@ class FairPoSCalculatorTest extends PropSpec with Matchers { """.stripMargin ) - val perfomances = calcPerfomance(chain, miners) - - println(perfomances.toList.sortBy(_._1).mkString("\n")) + val minersPerfomance = calcPerfomance(chain, miners) - assert(perfomances.forall(p => p._2 < 1.1 && p._2 > 0.9)) + assert(minersPerfomance.forall(p => p._2 < 1.1 && p._2 > 0.9)) assert(avgDelay < 80000 && avgDelay > 40000) assert(avgBT < 200 && avgBT > 20) } def mineBlock(prev: Block, grand: Option[Block], minerWithBalance: (PrivateKeyAccount, Long)): Block = { val (miner, balance) = minerWithBalance - val gs = pos.generatorSignature(generationSignature, miner.publicKey) - val hit = pos.hit(gs) - val delay = pos.calculateDelay(hit, prev.baseTarget, balance) - val bt = pos.baseTarget( + val gs = generatorSignature(generationSignature, miner.publicKey) + val h = hit(gs) + val delay = pos.calculateDelay(h, prev.baseTarget, balance) + val bt = pos.calculateBaseTarget( blockDelaySeconds, prev.height + 1, prev.baseTarget, From da20531f0c604f455d0c418acf74588ffde0e163 Mon Sep 17 00:00:00 2001 From: peterz Date: Wed, 23 May 2018 16:31:47 +0300 Subject: [PATCH 37/52] NODE-767 /utils/script API methods always return HTTP_OK --- src/main/scala/scorex/api/http/ApiError.scala | 6 ++++++ src/main/scala/scorex/api/http/UtilsApiRoute.scala | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/scala/scorex/api/http/ApiError.scala b/src/main/scala/scorex/api/http/ApiError.scala index d8340d3495c..616b01e4555 100755 --- a/src/main/scala/scorex/api/http/ApiError.scala +++ b/src/main/scala/scorex/api/http/ApiError.scala @@ -215,3 +215,9 @@ case object DataKeyNotExists extends ApiError { override val code = StatusCodes.NotFound override val message: String = "no data for this key" } + +case class ScriptCompilerError(errorMessage: String) extends ApiError { + override val id: Int = 305 + override val code: StatusCode = StatusCodes.BadRequest + override val message: String = errorMessage +} diff --git a/src/main/scala/scorex/api/http/UtilsApiRoute.scala b/src/main/scala/scorex/api/http/UtilsApiRoute.scala index fe5b759a4ca..730478a2122 100755 --- a/src/main/scala/scorex/api/http/UtilsApiRoute.scala +++ b/src/main/scala/scorex/api/http/UtilsApiRoute.scala @@ -43,7 +43,7 @@ case class UtilsApiRoute(timeService: Time, settings: RestAPISettings) extends A (post & entity(as[String])) { code => complete( ScriptCompiler(code).fold( - e => Json.obj("error" -> e), { + e => ScriptCompilerError(e), { case (script, complexity) => Json.obj( "script" -> script.bytes().base64, @@ -77,7 +77,7 @@ case class UtilsApiRoute(timeService: Time, settings: RestAPISettings) extends A ScriptCompiler.estimate(script).map((script, _)) } .fold( - e => Json.obj("error" -> e), { + e => ScriptCompilerError(e), { case (script, complexity) => Json.obj( "script" -> code, From 3e434070e8f163f1d19a8dd0cf356559f309a80a Mon Sep 17 00:00:00 2001 From: Vyatcheslav Suharnikov Date: Fri, 11 May 2018 19:03:23 +0300 Subject: [PATCH 38/52] NODE-742 * INVALID expression; * Parser returns multiple expressions; * Parsing valid and invalid names; * Improved parsing of strings (including unicode and special symbols); * Parsing of invalid string sequences; * Improved a error handling in parsing of escaped symbols; * Handling of invalid Base58 string; * Lightweight definition of expressions with PART; * Additional tests for CompilerV1; * JsAPI.parse/compile works with sequences; * Handling invalid MATCH_CASE; * Working with invalid function calls. --- .../state/StateSyntheticBenchmark.scala | 3 +- .../it/async/RollbackSpecSuite.scala | 1 - .../sync/AtomicSwapSmartContractSuite.scala | 10 +- .../sync/LeaseSmartContractsTestSuite.scala | 3 +- .../sync/MassTransferSmartContractSuite.scala | 3 +- .../sync/MassTransferTransactionSuite.scala | 5 +- .../it/sync/SetScriptTransactionSuite.scala | 5 +- .../sync/transactions/SponsorshipSuite.scala | 1 - lang/js/src/main/scala/JsAPI.scala | 7 +- .../wavesplatform/lang/IntegrationTest.scala | 14 +- .../com/wavesplatform/lang/ParserTest.scala | 411 ++++++++++++++---- .../lang/typechecker/CompilerV1Test.scala | 46 +- .../lang/typechecker/ErrorTest.scala | 20 +- .../lang/v1/compiler/CompilerV1.scala | 252 ++++++----- .../lang/v1/compiler/Terms.scala | 1 - .../lang/v1/parser/BinaryOperation.scala | 34 +- .../lang/v1/parser/Expressions.scala | 70 ++- .../wavesplatform/lang/v1/parser/Parser.scala | 201 ++++++--- .../lang/v1/parser/UnaryOperation.scala | 14 +- .../lang/v1/testing/ScriptGen.scala | 125 ++++-- .../diffs/AssetTransactionsDiffTest.scala | 3 +- .../SigVerifyPerformanceTest.scala | 3 +- .../state/diffs/smart/predef/package.scala | 3 +- .../AddressFromRecipientScenarioTest.scala | 3 +- .../scenarios/HackatonScenartioTest.scala | 14 +- .../smart/scenarios/LazyFieldAccessTest.scala | 3 +- .../smart/scenarios/MultiSig2of3Test.scala | 5 +- .../scenarios/OnlyTransferIsAllowedTest.scala | 7 +- .../smart/scenarios/OracleDataTest.scala | 13 +- 29 files changed, 887 insertions(+), 393 deletions(-) diff --git a/benchmark/src/test/scala/com/wavesplatform/state/StateSyntheticBenchmark.scala b/benchmark/src/test/scala/com/wavesplatform/state/StateSyntheticBenchmark.scala index 8d1fa793a59..a4ed16fb6f5 100644 --- a/benchmark/src/test/scala/com/wavesplatform/state/StateSyntheticBenchmark.scala +++ b/benchmark/src/test/scala/com/wavesplatform/state/StateSyntheticBenchmark.scala @@ -74,7 +74,8 @@ object StateSyntheticBenchmark { val textScript = "sigVerify(tx.bodyBytes,tx.proof0,tx.senderPk)" val untypedScript = Parser(textScript).get.value - val typedScript = CompilerV1(dummyTypeCheckerContext, untypedScript).explicitGet() + assert(untypedScript.size == 1) + val typedScript = CompilerV1(dummyTypeCheckerContext, untypedScript.head).explicitGet() val setScriptBlock = nextBlock( Seq( diff --git a/it/src/test/scala/com/wavesplatform/it/async/RollbackSpecSuite.scala b/it/src/test/scala/com/wavesplatform/it/async/RollbackSpecSuite.scala index 0585aa0eed5..250902d47e4 100644 --- a/it/src/test/scala/com/wavesplatform/it/async/RollbackSpecSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/async/RollbackSpecSuite.scala @@ -14,7 +14,6 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future.traverse import scala.concurrent.duration._ import scala.concurrent.{Await, Future} -import scala.util.Random class RollbackSpecSuite extends FreeSpec diff --git a/it/src/test/scala/com/wavesplatform/it/sync/AtomicSwapSmartContractSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/AtomicSwapSmartContractSuite.scala index dddf88bb790..bdb69f2fbeb 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/AtomicSwapSmartContractSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/AtomicSwapSmartContractSuite.scala @@ -16,8 +16,8 @@ import scorex.transaction.smart.SetScriptTransaction import scorex.transaction.smart.script.v1.ScriptV1 import scorex.transaction.transfer._ -import scala.util.Random import scala.concurrent.duration._ +import scala.util.Random /* Scenario: @@ -26,7 +26,6 @@ Scenario: 3. Alice funds swapBC1t 4. Alice can't take money from swapBC1 5.1 Bob takes funds because he knows secret hash OR 5.2 Wait height and Alice takes funds back - */ class AtomicSwapSmartContractSuite extends BaseTransactionSuite with CancelAfterFailure { @@ -55,8 +54,8 @@ class AtomicSwapSmartContractSuite extends BaseTransactionSuite with CancelAfter val beforeHeight = sender.height val sc1 = { val untyped = Parser(s""" - let Bob = extract(addressFromString("${BobBC1}")).bytes - let Alice = extract(addressFromString("${AliceBC1}")).bytes + let Bob = extract(addressFromString("$BobBC1")).bytes + let Alice = extract(addressFromString("$AliceBC1")).bytes let AlicesPK = base58'${ByteStr(AlicesPK.publicKey)}' let txRecipient = addressFromRecipient(tx.recipient).bytes @@ -67,7 +66,8 @@ class AtomicSwapSmartContractSuite extends BaseTransactionSuite with CancelAfter txToBob || backToAliceAfterHeight """.stripMargin).get.value - CompilerV1(dummyTypeCheckerContext, untyped).explicitGet() + assert(untyped.size == 1) + CompilerV1(dummyTypeCheckerContext, untyped.head).explicitGet() } val pkSwapBC1 = PrivateKeyAccount.fromSeed(sender.seed(swapBC1)).right.get diff --git a/it/src/test/scala/com/wavesplatform/it/sync/LeaseSmartContractsTestSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/LeaseSmartContractsTestSuite.scala index a77aa3de231..5ac4c780495 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/LeaseSmartContractsTestSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/LeaseSmartContractsTestSuite.scala @@ -47,7 +47,8 @@ class LeaseSmartContractsTestSuite extends BaseTransactionSuite with CancelAfter leaseTx || leaseCancelTx """.stripMargin).get.value - CompilerV1(dummyTypeCheckerContext, sc).explicitGet() + assert(sc.size == 1) + CompilerV1(dummyTypeCheckerContext, sc.head).explicitGet() } val script = ScriptV1(scriptText).explicitGet() diff --git a/it/src/test/scala/com/wavesplatform/it/sync/MassTransferSmartContractSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/MassTransferSmartContractSuite.scala index 58b195af98a..b129400834c 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/MassTransferSmartContractSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/MassTransferSmartContractSuite.scala @@ -51,7 +51,8 @@ class MassTransferSmartContractSuite extends BaseTransactionSuite with CancelAft (txToGovComplete && accSig && txToGov) || (txToUsers && accSig) """.stripMargin).get.value - CompilerV1(dummyTypeCheckerContext, untyped).explicitGet() + assert(untyped.size == 1) + CompilerV1(dummyTypeCheckerContext, untyped.head).explicitGet() } // set script diff --git a/it/src/test/scala/com/wavesplatform/it/sync/MassTransferTransactionSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/MassTransferTransactionSuite.scala index d08a8c45a02..2c2f4dc75dd 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/MassTransferTransactionSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/MassTransferTransactionSuite.scala @@ -3,12 +3,11 @@ package com.wavesplatform.it.sync import com.wavesplatform.it.api.SyncHttpApi._ import com.wavesplatform.it.transactions.BaseTransactionSuite import com.wavesplatform.it.util._ +import com.wavesplatform.utils.Base58 import org.scalatest.CancelAfterFailure import play.api.libs.json._ import scorex.api.http.assets.{MassTransferRequest, SignedMassTransferRequest} -import com.wavesplatform.utils.Base58 -import scorex.transaction.transfer.MassTransferTransaction.Transfer -import scorex.transaction.transfer.MassTransferTransaction.MaxTransferCount +import scorex.transaction.transfer.MassTransferTransaction.{MaxTransferCount, Transfer} import scorex.transaction.transfer.TransferTransaction.MaxAttachmentSize import scorex.transaction.transfer._ diff --git a/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala index d318bb4349e..1bea3b151f9 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/SetScriptTransactionSuite.scala @@ -12,9 +12,9 @@ import org.scalatest.CancelAfterFailure import play.api.libs.json.{JsNumber, Json} import scorex.account.PrivateKeyAccount import scorex.transaction.Proofs -import scorex.transaction.transfer._ import scorex.transaction.smart.SetScriptTransaction import scorex.transaction.smart.script.v1.ScriptV1 +import scorex.transaction.transfer._ class SetScriptTransactionSuite extends BaseTransactionSuite with CancelAfterFailure { private def pkFromAddress(address: String) = PrivateKeyAccount.fromSeed(sender.seed(address)).right.get @@ -64,7 +64,8 @@ class SetScriptTransactionSuite extends BaseTransactionSuite with CancelAfterFai AC && BC """.stripMargin).get.value - CompilerV1(dummyTypeCheckerContext, untyped).explicitGet() + assert(untyped.size == 1) + CompilerV1(dummyTypeCheckerContext, untyped.head).explicitGet() } val script = ScriptV1(scriptText).explicitGet() diff --git a/it/src/test/scala/com/wavesplatform/it/sync/transactions/SponsorshipSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/transactions/SponsorshipSuite.scala index 43e5a7b1fe9..269d6bf649a 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/transactions/SponsorshipSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/transactions/SponsorshipSuite.scala @@ -49,7 +49,6 @@ class SponsorshipSuite extends FreeSpec with NodesFromDocker with Matchers with val sponsorWavesBalance = miner.accountBalances(sponsor.address)._2 val sponsorAssetTotal = 100 * Token val minerWavesBalance = miner.accountBalances(miner.address)._2 - val aliceWavesBalance = miner.accountBalances(alice.address)._2 val sponsorAssetId = sponsor diff --git a/lang/js/src/main/scala/JsAPI.scala b/lang/js/src/main/scala/JsAPI.scala index 7c50b7bd9cd..e7364a0a6c5 100644 --- a/lang/js/src/main/scala/JsAPI.scala +++ b/lang/js/src/main/scala/JsAPI.scala @@ -8,13 +8,16 @@ import scala.scalajs.js.annotation.JSExportTopLevel object JsAPI { @JSExportTopLevel("parse") - def parse(input: String): Parsed[Expressions.EXPR, Char, String] = + def parse(input: String): Parsed[Seq[Expressions.EXPR], Char, String] = Parser(input) @JSExportTopLevel("compile") def compile(input: String): Option[BitVector] = { parse(input) - .fold[Option[Expressions.EXPR]]((_, _, _) => None, (x, _) => Some(x)) + .fold[Option[Expressions.EXPR]]((_, _, _) => None, { (xs, _) => + if (xs.size == 1) Some(xs.head) + else None + }) .flatMap(CompilerV1(CompilerContext.empty, _).toOption) .flatMap(Serde.codec.encode(_).toOption) } diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala index 286e26f271d..455777b67a9 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala @@ -18,13 +18,10 @@ class IntegrationTest extends PropSpec with PropertyChecks with ScriptGen with M property("patternMatching") { val sampleScript = - """| - |match p { - | case pa: PointA => 0 - | case pb: PointB => 1 - |} - | - """.stripMargin + """match p { + | case pa: PointA => 0 + | case pb: PointB => 1 + |}""".stripMargin eval[Long](sampleScript, withUnion(pointAInstance)) shouldBe Right(0) eval[Long](sampleScript, withUnion(pointBInstance)) shouldBe Right(1) } @@ -57,7 +54,8 @@ class IntegrationTest extends PropSpec with PropertyChecks with ScriptGen with M private def eval[T: TypeInfo](code: String, ctx: EvaluationContext = PureContext.instance) = { val untyped = Parser(code).get.value - val typed = CompilerV1(CompilerContext.fromEvaluationContext(ctx), untyped) + require(untyped.size == 1) + val typed = CompilerV1(CompilerContext.fromEvaluationContext(ctx), untyped.head) typed.flatMap(EvaluatorV1[T](ctx, _)) } diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala index 990c18b0593..8b77907ffd4 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala @@ -15,7 +15,20 @@ import scorex.crypto.encode.{Base58 => ScorexBase58} class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptGenParser with NoShrink { - def parse(x: String): EXPR = Parser(x) match { + private def parseOne(x: String): EXPR = Parser(x) match { + case Success(r, _) => + if (r.size > 1) { + println(s"Can't parse (len=${x.length}): \n$x\n") + throw new TestFailedException(s"Expected 1 expression, but got ${r.size}: $r", 0) + } else r.head + case e @ Failure(_, i, _) => + println( + s"Can't parse (len=${x.length}): \n$x\n\nError: $e\nPosition ($i): '${x.slice(i, i + 1)}'\nTraced:\n${e.extra.traced.fullStack + .mkString("\n")}") + throw new TestFailedException("Test failed", 0) + } + + private def parseAll(x: String): Seq[EXPR] = Parser(x) match { case Success(r, _) => r case e @ Failure(_, i, _) => println(x) @@ -25,49 +38,62 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG throw new TestFailedException("Test failed", 0) } - def isParsed(x: String): Boolean = Parser(x) match { + private def isParsed(x: String): Boolean = Parser(x) match { case Success(_, _) => true case Failure(_, _, _) => false } - def genElementCheck(gen: Gen[EXPR]): Unit = { + private def genElementCheck(gen: Gen[EXPR]): Unit = { val testGen: Gen[(EXPR, String)] = for { expr <- gen str <- toString(expr) } yield (expr, str) forAll(testGen) { - case ((expr, str)) => - parse(str) shouldBe expr + case (expr, str) => + withClue(str) { + parseOne(str) shouldBe expr + } } } - property("all types of multiline expressions") { - val gas = 50 - genElementCheck(CONST_LONGgen.map(_._1)) - genElementCheck(STRgen) - genElementCheck(REFgen) - genElementCheck(BOOLgen(gas).map(_._1)) - genElementCheck(SUMgen(gas).map(_._1)) - genElementCheck(EQ_INTgen(gas).map(_._1)) - genElementCheck(INTGen(gas).map(_._1)) - genElementCheck(GEgen(gas).map(_._1)) - genElementCheck(GTgen(gas).map(_._1)) - genElementCheck(ANDgen(gas).map(_._1)) - genElementCheck(ORgen(gas).map(_._1)) - genElementCheck(BLOCKgen(gas)) + private def multiLineExprTests(tests: (String, Gen[EXPR])*): Unit = tests.foreach { + case (label, gen) => + property(s"multiline expressions: $label") { + genElementCheck(gen) + } } + private val gas = 50 + multiLineExprTests( + "CONST_LONG" -> CONST_LONGgen.map(_._1), + "STR" -> STRgen, + "REF" -> REFgen, + "BOOL" -> BOOLgen(gas).map(_._1), + "SUM" -> SUMgen(gas).map(_._1), + "EQ" -> EQ_INTgen(gas).map(_._1), + "INT" -> INTGen(gas).map(_._1), + "GE" -> GEgen(gas).map(_._1), + "GT" -> GTgen(gas).map(_._1), + "AND" -> ANDgen(gas).map(_._1), + "OR" -> ORgen(gas).map(_._1), + "BLOCK" -> BLOCKgen(gas) + ) + property("priority in binary expressions") { - parse("1 == 0 || 3 == 2") shouldBe BINARY_OP(BINARY_OP(CONST_LONG(1), EQ_OP, CONST_LONG(0)), - OR_OP, - BINARY_OP(CONST_LONG(3), EQ_OP, CONST_LONG(2))) - parse("3 + 2 > 2 + 1") shouldBe BINARY_OP(BINARY_OP(CONST_LONG(3), SUM_OP, CONST_LONG(2)), GT_OP, BINARY_OP(CONST_LONG(2), SUM_OP, CONST_LONG(1))) - parse("1 >= 0 || 3 > 2") shouldBe BINARY_OP(BINARY_OP(CONST_LONG(1), GE_OP, CONST_LONG(0)), OR_OP, BINARY_OP(CONST_LONG(3), GT_OP, CONST_LONG(2))) + parseOne("1 == 0 || 3 == 2") shouldBe BINARY_OP(BINARY_OP(CONST_LONG(1), EQ_OP, CONST_LONG(0)), + OR_OP, + BINARY_OP(CONST_LONG(3), EQ_OP, CONST_LONG(2))) + parseOne("3 + 2 > 2 + 1") shouldBe BINARY_OP(BINARY_OP(CONST_LONG(3), SUM_OP, CONST_LONG(2)), + GT_OP, + BINARY_OP(CONST_LONG(2), SUM_OP, CONST_LONG(1))) + parseOne("1 >= 0 || 3 > 2") shouldBe BINARY_OP(BINARY_OP(CONST_LONG(1), GE_OP, CONST_LONG(0)), + OR_OP, + BINARY_OP(CONST_LONG(3), GT_OP, CONST_LONG(2))) } property("bytestr expressions") { - parse("false || sigVerify(base58'333', base58'222', base58'111')") shouldBe BINARY_OP( + parseOne("false || sigVerify(base58'333', base58'222', base58'111')") shouldBe BINARY_OP( FALSE, OR_OP, FUNCTION_CALL( @@ -81,20 +107,26 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG ) } - property("base58") { - parse("base58'bQbp'") shouldBe CONST_BYTEVECTOR(ByteVector("foo".getBytes)) - parse("base58''") shouldBe CONST_BYTEVECTOR(ByteVector.empty) - isParsed("base58' bQbp'\n") shouldBe false + property("valid non-empty base58 definition") { + parseOne("base58'bQbp'") shouldBe CONST_BYTEVECTOR(ByteVector("foo".getBytes)) + } + + property("valid empty base58 definition") { + parseOne("base58''") shouldBe CONST_BYTEVECTOR(ByteVector.empty) + } + + property("invalid base58 definition") { + parseOne("base58' bQbp'") shouldBe CONST_BYTEVECTOR(PART.INVALID(" bQbp", "Can't parse Base58 string")) } property("string is consumed fully") { - parse(""" " fooo bar" """) shouldBe CONST_STRING(" fooo bar") + parseOne(""" " fooo bar" """) shouldBe CONST_STRING(" fooo bar") } property("string literal with unicode chars") { val stringWithUnicodeChars = "❤✓☀★☂♞☯☭☢€☎∞❄♫\u20BD" - parse( + parseOne( s""" | | "$stringWithUnicodeChars" @@ -103,16 +135,45 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG ) shouldBe CONST_STRING(stringWithUnicodeChars) } + property("string literal with unicode chars in language") { + parseOne("\"\\u1234\"") shouldBe CONST_STRING("ሴ") + } + + property("should parse invalid unicode symbols") { + parseOne("\"\\uqwer\"") shouldBe CONST_STRING(PART.INVALID("\\uqwer", "Can't parse 'qwer' as HEX string in '\\uqwer'")) + } + + property("should parse incomplete unicode symbol definition") { + parseOne("\"\\u12 test\"") shouldBe CONST_STRING(PART.INVALID("\\u12 test", "Incomplete UTF-8 symbol definition: '\\u12'")) + parseOne("\"\\u\"") shouldBe CONST_STRING(PART.INVALID("\\u", "Incomplete UTF-8 symbol definition: '\\u'")) + } + + property("string literal with special symbols") { + parseOne("\"\\t\"") shouldBe CONST_STRING("\t") + } + + property("should parse invalid special symbols") { + parseOne("\"\\ test\"") shouldBe CONST_STRING(PART.INVALID("\\ test", "Unknown escaped symbol: '\\ '")) + } + + property("should parse incomplete special symbols") { + parseOne("\"foo \\\"") shouldBe CONST_STRING(PART.INVALID("foo \\", "Invalid escaped symbol: '\\'")) + } + property("reserved keywords are invalid variable names") { - def script(keyword: String): String = - s""" - | - |let $keyword = 1 - |$keyword + 1 - | - """.stripMargin + List("if", "then", "else", "true", "false", "let").foreach { keyword => + val script = s"""let $keyword = 1 + |true""".stripMargin + parseOne(script) shouldBe BLOCK( + LET(PART.INVALID(keyword, "keywords are restricted"), CONST_LONG(1), Seq.empty), + TRUE + ) + } - List("if", "then", "else", "true", "false", "let").foreach(kv => isParsed(script(kv)) shouldBe false) + List("if", "then", "else", "let").foreach { keyword => + val script = s"$keyword + 1" + parseOne(script) shouldBe BINARY_OP(REF(PART.INVALID(keyword, "keywords are restricted")), BinaryOperation.SUM_OP, CONST_LONG(1)) + } } property("multisig sample") { @@ -134,27 +195,43 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG | AC + BC+ CC >= 2 | """.stripMargin - parse(script) // gets parsed, but later will fail on type check! + parseOne(script) // gets parsed, but later will fail on type check! } property("function call") { - parse("FOO(1,2)".stripMargin) shouldBe FUNCTION_CALL(("FOO"), List(CONST_LONG(1), CONST_LONG(2))) - parse("FOO(X)".stripMargin) shouldBe FUNCTION_CALL(("FOO"), List(REF("X"))) + parseOne("FOO(1,2)".stripMargin) shouldBe FUNCTION_CALL("FOO", List(CONST_LONG(1), CONST_LONG(2))) + parseOne("FOO(X)".stripMargin) shouldBe FUNCTION_CALL("FOO", List(REF("X"))) + } + + property("function call on curly braces") { + parseOne("{ 1 }(2, 3, 4)") shouldBe FUNCTION_CALL( + PART.INVALID("", "CONST_LONG(1) is not a function name"), + List(2, 3, 4).map(CONST_LONG(_)) + ) + } + + property("function call on round braces") { + parseOne("(1)(2, 3, 4)") shouldBe FUNCTION_CALL( + PART.INVALID("", "CONST_LONG(1) is not a function name"), + List(2, 3, 4).map(CONST_LONG(_)) + ) } property("isDefined/extract") { - parse("isDefined(X)") shouldBe FUNCTION_CALL(("isDefined"), List(REF("X"))) - parse("if(isDefined(X)) then extract(X) else Y") shouldBe IF(FUNCTION_CALL(("isDefined"), List(REF("X"))), - FUNCTION_CALL(("extract"), List(REF("X"))), - REF("Y")) + parseOne("isDefined(X)") shouldBe FUNCTION_CALL("isDefined", List(REF("X"))) + parseOne("if(isDefined(X)) then extract(X) else Y") shouldBe IF( + FUNCTION_CALL("isDefined", List(REF("X"))), + FUNCTION_CALL("extract", List(REF("X"))), + REF("Y") + ) } property("getter") { - isParsed("xxx .yyy") shouldBe false - isParsed("xxx. yyy") shouldBe false + isParsed("xxx .yyy") shouldBe true + isParsed("xxx. yyy") shouldBe true - parse("xxx.yyy") shouldBe GETTER(REF("xxx"), "yyy") - parse( + parseOne("xxx.yyy") shouldBe GETTER(REF("xxx"), "yyy") + parseOne( """ | | xxx.yyy @@ -162,26 +239,26 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG """.stripMargin ) shouldBe GETTER(REF("xxx"), "yyy") - parse("xxx(yyy).zzz") shouldBe GETTER(FUNCTION_CALL(("xxx"), List(REF("yyy"))), "zzz") - parse( + parseOne("xxx(yyy).zzz") shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") + parseOne( """ | | xxx(yyy).zzz | """.stripMargin - ) shouldBe GETTER(FUNCTION_CALL(("xxx"), List(REF("yyy"))), "zzz") + ) shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") - parse("(xxx(yyy)).zzz") shouldBe GETTER(FUNCTION_CALL(("xxx"), List(REF("yyy"))), "zzz") - parse( + parseOne("(xxx(yyy)).zzz") shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") + parseOne( """ | | (xxx(yyy)).zzz | """.stripMargin - ) shouldBe GETTER(FUNCTION_CALL(("xxx"), List(REF("yyy"))), "zzz") + ) shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") - parse("{xxx(yyy)}.zzz") shouldBe GETTER(FUNCTION_CALL(("xxx"), List(REF("yyy"))), "zzz") - parse( + parseOne("{xxx(yyy)}.zzz") shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") + parseOne( """ | | { @@ -189,9 +266,9 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG | }.zzz | """.stripMargin - ) shouldBe GETTER(FUNCTION_CALL(("xxx"), List(REF("yyy"))), "zzz") + ) shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") - parse( + parseOne( """ | | { @@ -200,7 +277,13 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG | }.zzz | """.stripMargin - ) shouldBe GETTER(BLOCK(LET("yyy", FUNCTION_CALL(("aaa"), List(REF("bbb")))), FUNCTION_CALL(("xxx"), List(REF("yyy")))), "zzz") + ) shouldBe GETTER( + BLOCK( + LET("yyy", FUNCTION_CALL("aaa", List(REF("bbb"))), Seq.empty), + FUNCTION_CALL("xxx", List(REF("yyy"))) + ), + "zzz" + ) } property("crypto functions") { @@ -209,13 +292,7 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG val encodedText = ScorexBase58.encode(text.getBytes) for (f <- hashFunctions) { - parse( - s""" - | - |$f(base58'$encodedText') - | - """.stripMargin - ) shouldBe + parseOne(s"$f(base58'$encodedText')".stripMargin) shouldBe FUNCTION_CALL( f, List(CONST_BYTEVECTOR(ByteVector(text.getBytes))) @@ -223,11 +300,60 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG } } - property("multiple expressions going one after another are denied") { - isParsed( - """1 + 1 - |2 + 2""".stripMargin - ) shouldBe false + property("show parse all input including INVALID") { + val script = + """let C = 1 + |foo + |#@2 + |true""".stripMargin + + parseAll(script) shouldBe Seq( + BLOCK( + LET("C", CONST_LONG(1), Seq.empty), + REF("foo") + ), + INVALID("#@", CONST_LONG(2)), + TRUE + ) + } + + property("should parse INVALID expressions in the middle") { + val script = + """let C = 1 + |# / + |true""".stripMargin + parseOne(script) shouldBe BLOCK( + LET("C", CONST_LONG(1), Seq.empty), + INVALID("#/", TRUE) + ) + } + + property("should parse INVALID expressions at start") { + val script = + """# / + |let C = 1 + |true""".stripMargin + parseOne(script) shouldBe INVALID( + "#/", + BLOCK( + LET("C", CONST_LONG(1), Seq.empty), + TRUE + ) + ) + } + + property("should parse INVALID expressions at end") { + val script = + """let C = 1 + |true + |# /""".stripMargin + parseAll(script) shouldBe Seq( + BLOCK( + LET("C", CONST_LONG(1), Seq.empty), + TRUE + ), + INVALID("#/") + ) } property("simple matching") { @@ -240,8 +366,8 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG | } | """.stripMargin - parse(code) shouldBe MATCH(REF("tx"), - List(MATCH_CASE(Some("a"), List("TypeA"), CONST_LONG(0)), MATCH_CASE(Some("b"), List("TypeB"), CONST_LONG(1)))) + parseOne(code) shouldBe MATCH(REF("tx"), + List(MATCH_CASE(Some("a"), List("TypeA"), CONST_LONG(0)), MATCH_CASE(Some("b"), List("TypeB"), CONST_LONG(1)))) } property("multiple union type matching") { @@ -254,9 +380,9 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG | } | """.stripMargin - parse(code) shouldBe MATCH(REF("tx"), - List(MATCH_CASE(Some("txa"), List("TypeA"), CONST_LONG(0)), - MATCH_CASE(Some("underscore"), List("TypeB", "TypeC"), CONST_LONG(1)))) + parseOne(code) shouldBe MATCH(REF("tx"), + List(MATCH_CASE(Some("txa"), List("TypeA"), CONST_LONG(0)), + MATCH_CASE(Some("underscore"), List("TypeB", "TypeC"), CONST_LONG(1)))) } property("matching expression") { @@ -269,29 +395,130 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG | } | """.stripMargin - parse(code) shouldBe MATCH( + parseOne(code) shouldBe MATCH( BINARY_OP(FUNCTION_CALL("foo", List(REF("x"))), BinaryOperation.SUM_OP, REF("bar")), List(MATCH_CASE(Some("x"), List("TypeA"), CONST_LONG(0)), MATCH_CASE(Some("y"), List("TypeB", "TypeC"), CONST_LONG(1))) ) } - property("matching with rest types declaration") { - parse("match tx { case x => 1 } ") shouldBe MATCH(REF("tx"), List(MATCH_CASE(Some("x"), List.empty, CONST_LONG(1)))) + property("pattern matching with valid case, but no type is defined") { + parseOne("match tx { case x => 1 } ") shouldBe MATCH( + REF("tx"), + List( + MATCH_CASE( + Some(PART.VALID("x")), + List.empty, + CONST_LONG(1) + ) + ) + ) + } + + property("pattern matching with valid case, placeholder instead of variable name") { + parseOne("match tx { case _:TypeA => 1 } ") shouldBe MATCH( + REF("tx"), + List( + MATCH_CASE( + None, + List(PART.VALID("TypeA")), + CONST_LONG(1) + ) + ) + ) } - property("matching with no variable declaration") { - parse("match tx { case _ : TypeA => 1 } ") shouldBe MATCH(REF("tx"), List(MATCH_CASE(None, List("TypeA"), CONST_LONG(1)))) + property("pattern matching with no cases") { + parseOne("match tx { } ") shouldBe INVALID("pattern matching requires case branches") + } + + property("pattern matching with invalid case - no variable, type and expr are defined") { + parseOne("match tx { case => } ") shouldBe MATCH( + REF("tx"), + List( + MATCH_CASE( + Some(PART.INVALID("", "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), + List.empty, + INVALID("expected expression") + ) + ) + ) } - property("failure to match") { - isParsed("match tx { } ") shouldBe false - isParsed("match tx { case => } ") shouldBe false - isParsed("match tx { case => 1} ") shouldBe false - isParsed("match tx { case TypeA => } ") shouldBe false - isParsed("match tx { case :TypeA => 1 } ") shouldBe false + property("pattern matching with invalid case - no variable and type are defined") { + parseOne("match tx { case => 1 } ") shouldBe MATCH( + REF("tx"), + List( + MATCH_CASE( + Some(PART.INVALID("", "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), + List.empty, + CONST_LONG(1) + ) + ) + ) + } - isParsed("match tx { case _:TypeA => 1 } ") shouldBe true - isParsed("match tx { case _: | => 1 } ") shouldBe false - isParsed("match tx { case _: |||| => 1 } ") shouldBe false + property("pattern matching with invalid case - no expr is defined") { + parseOne("match tx { case TypeA => } ") shouldBe MATCH( + REF("tx"), + List( + MATCH_CASE( + Some(PART.VALID("TypeA")), + Seq.empty, + INVALID("expected expression") + ) + ) + ) + } + + property("pattern matching with invalid case - no var is defined") { + parseOne("match tx { case :TypeA => 1 } ") shouldBe MATCH( + REF("tx"), + List( + MATCH_CASE( + Some(PART.INVALID(":TypeA ", "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), + Seq.empty, + CONST_LONG(1) + ) + ) + ) + } + + property("pattern matching with invalid case - expression in variable definition") { + parseOne("match tx { case 1 + 1 => 1 } ") shouldBe MATCH( + REF("tx"), + List( + MATCH_CASE( + Some(PART.INVALID("1 + 1 ", "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), + List.empty, + CONST_LONG(1) + ) + ) + ) + } + + property("pattern matching with default case - no type is defined, one separator") { + parseOne("match tx { case _: | => 1 } ") shouldBe MATCH( + REF("tx"), + List( + MATCH_CASE( + None, + Seq(PART.INVALID("| ", "the type for variable should be specified: `case varName: Type => expr`")), + CONST_LONG(1) + ) + ) + ) + } + + property("pattern matching with default case - no type is defined, multiple separators") { + parseOne("match tx { case _: |||| => 1 } ") shouldBe MATCH( + REF("tx"), + List( + MATCH_CASE( + None, + Seq(PART.INVALID("|||| ", "the type for variable should be specified: `case varName: Type => expr`")), + CONST_LONG(1) + ) + ) + ) } } diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala index ac76f5f94ad..ffec93c6878 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala @@ -1,10 +1,10 @@ package com.wavesplatform.lang.typechecker import com.wavesplatform.lang.Common._ -import com.wavesplatform.lang.v1.compiler.Terms._ import com.wavesplatform.lang.v1.compiler -import com.wavesplatform.lang.v1.compiler.{CompilerContext, CompilerV1} import com.wavesplatform.lang.v1.compiler.CompilerV1.CompilationResult +import com.wavesplatform.lang.v1.compiler.Terms._ +import com.wavesplatform.lang.v1.compiler.{CompilerContext, CompilerV1} import com.wavesplatform.lang.v1.evaluator.ctx.impl.PureContext._ import com.wavesplatform.lang.v1.parser.BinaryOperation.SUM_OP import com.wavesplatform.lang.v1.parser.Expressions @@ -104,6 +104,48 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr ) ) + treeTypeTest("Invalid LET")( + ctx = typeCheckerContext, + expr = Expressions.BLOCK(Expressions.LET(Expressions.PART.INVALID("###", "it is invalid!"), Expressions.TRUE, Seq.empty), Expressions.REF("x")), + expectedResult = Left("Typecheck failed: it is invalid!: ###") + ) + + treeTypeTest("Invalid GETTER")( + ctx = typeCheckerContext, + expr = Expressions.GETTER(Expressions.REF("x"), Expressions.PART.INVALID("###", "it is invalid!")), + expectedResult = Left("Typecheck failed: it is invalid!: ###") + ) + + treeTypeTest("Invalid BYTEVECTOR")( + ctx = typeCheckerContext, + expr = Expressions.CONST_BYTEVECTOR(Expressions.PART.INVALID("foo", "it is invalid!")), + expectedResult = Left("Typecheck failed: it is invalid!: foo") + ) + + treeTypeTest("Invalid STRING")( + ctx = typeCheckerContext, + expr = Expressions.CONST_STRING(Expressions.PART.INVALID("\\u1", "it is invalid!")), + expectedResult = Left("Typecheck failed: it is invalid!: \\u1") + ) + + treeTypeTest("Invalid REF")( + ctx = typeCheckerContext, + expr = Expressions.REF(Expressions.PART.INVALID("###", "it is invalid!")), + expectedResult = Left("Typecheck failed: it is invalid!: ###") + ) + + treeTypeTest("Invalid FUNCTION_CALL")( + ctx = typeCheckerContext, + expr = Expressions.FUNCTION_CALL(Expressions.PART.INVALID("###", "it is invalid!"), List.empty), + expectedResult = Left("Typecheck failed: it is invalid!: ###") + ) + + treeTypeTest("INVALID")( + ctx = typeCheckerContext, + expr = Expressions.INVALID("###", None), + expectedResult = Left("Typecheck failed: ###") + ) + private def treeTypeTest(propertyName: String)(expr: Expressions.EXPR, expectedResult: CompilationResult[EXPR], ctx: CompilerContext): Unit = property(propertyName) { compiler.CompilerV1(ctx, expr) shouldBe expectedResult diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/ErrorTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/ErrorTest.scala index 8fcb7389b09..e16f473aec6 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/ErrorTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/ErrorTest.scala @@ -14,21 +14,23 @@ class ErrorTest extends PropSpec with PropertyChecks with Matchers with ScriptGe import com.wavesplatform.lang.v1.parser.Expressions._ errorTests( - "can't define LET with the same name as already defined in scope" -> "already defined in the scope" -> BLOCK(LET("X", CONST_LONG(1)), - BLOCK(LET("X", CONST_LONG(2)), - TRUE)), - "can't define LET with the same name as predefined constant" -> "already defined in the scope" -> BLOCK(LET("None", CONST_LONG(2)), TRUE), - "can't define LET with the same name as predefined function" -> "function with such name is predefined" -> BLOCK(LET("Some", CONST_LONG(2)), - TRUE), + "can't define LET with the same name as already defined in scope" -> "already defined in the scope" -> BLOCK( + LET("X", CONST_LONG(1), Seq.empty), + BLOCK(LET("X", CONST_LONG(2), Seq.empty), TRUE)), + "can't define LET with the same name as predefined constant" -> "already defined in the scope" -> BLOCK(LET("None", CONST_LONG(2), Seq.empty), + TRUE), + "can't define LET with the same name as predefined function" -> "function with such name is predefined" -> BLOCK( + LET("Some", CONST_LONG(2), Seq.empty), + TRUE), "BINARY_OP with wrong types" -> "Typecheck failed: Can't find a function '+'" -> BINARY_OP(TRUE, SUM_OP, CONST_LONG(1)), "IF can't find common" -> "Can't find common type" -> IF(TRUE, TRUE, CONST_LONG(0)), "IF clause must be boolean" -> "IF clause is expected to be BOOLEAN" -> IF(CONST_LONG(0), TRUE, FALSE), - "FUNCTION_CALL with wrong amount of arguments" -> "requires 2 arguments" -> FUNCTION_CALL((multiplierFunction.name), List(CONST_LONG(0))), - "FUNCTION_CALL with upper type" -> "Non-matching types" -> FUNCTION_CALL((unitOnNone.name), List(FUNCTION_CALL(("Some"), List(CONST_LONG(3))))), + "FUNCTION_CALL with wrong amount of arguments" -> "requires 2 arguments" -> FUNCTION_CALL(multiplierFunction.name, List(CONST_LONG(0))), + "FUNCTION_CALL with upper type" -> "Non-matching types" -> FUNCTION_CALL(unitOnNone.name, List(FUNCTION_CALL("Some", List(CONST_LONG(3))))), "FUNCTION_CALL with wrong type of argument" -> "Typecheck failed: Non-matching types: expected: LONG, actual: BOOLEAN" -> FUNCTION_CALL( multiplierFunction.name, List(CONST_LONG(0), FALSE)), - "FUNCTION_CALL with uncommon types for parameter T" -> "Can't match inferred types" -> FUNCTION_CALL((functionWithTwoPrarmsOfTheSameType.name), + "FUNCTION_CALL with uncommon types for parameter T" -> "Can't match inferred types" -> FUNCTION_CALL(functionWithTwoPrarmsOfTheSameType.name, List(CONST_LONG(1), CONST_BYTEVECTOR(ByteVector.empty))) ) diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala index 231961e5424..c7b51e9e7a9 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala @@ -1,6 +1,8 @@ package com.wavesplatform.lang.v1.compiler import cats.data._ +import cats.instances.either._ +import cats.instances.list._ import cats.syntax.all._ import com.wavesplatform.lang.ExprCompiler import com.wavesplatform.lang.ScriptVersion.Versions.V1 @@ -11,7 +13,7 @@ import com.wavesplatform.lang.v1.compiler.Terms._ import com.wavesplatform.lang.v1.evaluator.ctx.PredefFunction.FunctionTypeSignature import com.wavesplatform.lang.v1.evaluator.ctx.impl.PureContext import com.wavesplatform.lang.v1.parser.BinaryOperation._ -import com.wavesplatform.lang.v1.parser.Expressions.BINARY_OP +import com.wavesplatform.lang.v1.parser.Expressions.{BINARY_OP, PART} import com.wavesplatform.lang.v1.parser.{BinaryOperation, Expressions, Parser} import monix.eval.Coeval @@ -23,11 +25,14 @@ class CompilerV1(ctx: CompilerContext) extends ExprCompiler { override def compile(input: String, directives: List[Directive]): Either[String, version.ExprT] = { Parser(input) match { - case fastparse.core.Parsed.Success(value, _) => - CompilerV1(ctx, value) match { - case Left(err) => Left(err.toString) - case Right(expr) => Right(expr) - } + case fastparse.core.Parsed.Success(xs, _) => + if (xs.size > 1) Left("Too many expressions") + else if (xs.isEmpty) Left("No expression") + else + CompilerV1(ctx, xs.head) match { + case Left(err) => Left(err.toString) + case Right(expr) => Right(expr) + } case f @ fastparse.core.Parsed.Failure(_, _, _) => Left(f.toString) } } @@ -39,10 +44,12 @@ object CompilerV1 { type CompilationResult[T] = Either[TypeResolutionError, T] private type SetTypeResult[T] = EitherT[Coeval, String, T] + type ResolvedArgsResult = EitherT[Coeval, String, List[EXPR]] + private def compile(ctx: CompilerContext, t: SetTypeResult[Expressions.EXPR]): SetTypeResult[EXPR] = t.flatMap { case x: Expressions.CONST_LONG => EitherT.pure(CONST_LONG(x.value)) - case x: Expressions.CONST_BYTEVECTOR => EitherT.pure(CONST_BYTEVECTOR(x.value)) - case x: Expressions.CONST_STRING => EitherT.pure(CONST_STRING(x.value)) + case Expressions.CONST_BYTEVECTOR(p) => handlePart(p)(CONST_BYTEVECTOR) + case Expressions.CONST_STRING(p) => handlePart(p)(CONST_STRING) case Expressions.TRUE => EitherT.pure(TRUE) case Expressions.FALSE => EitherT.pure(FALSE) case getter: Expressions.GETTER => compileGetter(ctx, getter) @@ -57,37 +64,41 @@ object CompilerV1 { case OR_OP => compileIf(ctx, Expressions.IF(a, Expressions.TRUE, b)) case _ => compileFunctionCall(ctx, Expressions.FUNCTION_CALL(opsToFunctions(op), List(a, b))) } + case Expressions.INVALID(message, _) => EitherT.leftT[Coeval, EXPR](message) } private def compileGetter(ctx: CompilerContext, getter: Expressions.GETTER): SetTypeResult[EXPR] = - compile(ctx, EitherT.pure(getter.ref)) - .subflatMap { subExpr => - def getField(name: String): Either[String, GETTER] = { - val refTpe = ctx.predefTypes.get(name).map(Right(_)).getOrElse(Left(s"Undefined type: $name")) - val fieldTpe = refTpe.flatMap { ct => - val fieldTpe = ct.fields.collectFirst { case (fieldName, tpe) if fieldName == getter.field => tpe } - fieldTpe.map(Right(_)).getOrElse(Left(s"Undefined field $name.${getter.field}")) + for { + field <- EitherT.fromEither[Coeval](getter.field.toEither) + r <- compile(ctx, EitherT.pure(getter.ref)) + .subflatMap { subExpr => + def getField(name: String): Either[String, GETTER] = { + val refTpe = ctx.predefTypes.get(name).map(Right(_)).getOrElse(Left(s"Undefined type: $name")) + val fieldTpe = refTpe.flatMap { ct => + val fieldTpe = ct.fields.collectFirst { case (fieldName, tpe) if fieldName == field => tpe } + fieldTpe.map(Right(_)).getOrElse(Left(s"Undefined field $name.${getter.field}")) + } + fieldTpe.right.map(tpe => GETTER(expr = subExpr, field = field, tpe = tpe)) } - fieldTpe.right.map(tpe => GETTER(expr = subExpr, field = getter.field, tpe = tpe)) - } - subExpr.tpe match { - case typeRef: TYPEREF => getField(typeRef.name) - case typeRef: CASETYPEREF => getField(typeRef.name) - case union: UNION => - val x1 = union.l - .map(k => ctx.predefTypes(k.name)) - .map(predefType => predefType.fields.find(_._1 == getter.field)) - if (x1.contains(None)) Left(s"Undefined field ${getter.field} on $union") - else - TypeInferrer.findCommonType(x1.map(_.get._2)) match { - case Some(cT) => Right(GETTER(expr = subExpr, field = getter.field, tpe = cT)) - case None => Left(s"Undefined common type for field ${getter.field} on $union") - } + subExpr.tpe match { + case typeRef: TYPEREF => getField(typeRef.name) + case typeRef: CASETYPEREF => getField(typeRef.name) + case union: UNION => + val x1 = union.l + .map(k => ctx.predefTypes(k.name)) + .map(predefType => predefType.fields.find(_._1 == getter.field)) + if (x1.contains(None)) Left(s"Undefined field ${getter.field} on $union") + else + TypeInferrer.findCommonType(x1.map(_.get._2)) match { + case Some(cT) => Right(GETTER(expr = subExpr, field = field, tpe = cT)) + case None => Left(s"Undefined common type for field ${getter.field} on $union") + } - case x => Left(s"Can't access to '${getter.field}' of a primitive type $x") + case x => Left(s"Can't access to '${getter.field}' of a primitive type $x") + } } - } + } yield r private def compileIf(ctx: CompilerContext, ifExpr: Expressions.IF): SetTypeResult[EXPR] = (compile(ctx, EitherT.pure(ifExpr.cond)), compile(ctx, EitherT.pure(ifExpr.ifTrue)), compile(ctx, EitherT.pure(ifExpr.ifFalse))).tupled @@ -114,87 +125,63 @@ object CompilerV1 { private def compileFunctionCall(ctx: CompilerContext, fc: Expressions.FUNCTION_CALL): SetTypeResult[EXPR] = { val Expressions.FUNCTION_CALL(name, args) = fc - type ResolvedArgsResult = EitherT[Coeval, String, List[EXPR]] - - def resolvedArguments(args: List[Expressions.EXPR]): ResolvedArgsResult = { - import cats.instances.list._ - val r: List[SetTypeResult[EXPR]] = args.map(arg => compile(ctx, EitherT.pure(arg)))(collection.breakOut) - r.sequence[SetTypeResult, EXPR] - } - - def matchOverload(resolvedArgs: List[EXPR], f: FunctionTypeSignature): Either[String, EXPR] = { - val argTypes = f.args - val resultType = f.result - if (args.lengthCompare(argTypes.size) != 0) - Left(s"Function '$name' requires ${argTypes.size} arguments, but ${args.size} are provided") - else { - val typedExpressionArgumentsAndTypedPlaceholders: List[(EXPR, TYPEPLACEHOLDER)] = resolvedArgs.zip(argTypes) - - val typePairs = typedExpressionArgumentsAndTypedPlaceholders.map { case (typedExpr, tph) => (typedExpr.tpe, tph) } - for { - resolvedTypeParams <- TypeInferrer(typePairs) - resolvedResultType <- TypeInferrer.inferResultType(resultType, resolvedTypeParams) - } yield - FUNCTION_CALL( - FunctionHeader(name, f.args.map(FunctionHeaderType.fromTypePlaceholder)), - typedExpressionArgumentsAndTypedPlaceholders.map(_._1), - resolvedResultType - ) - } - } + for { + name <- EitherT.fromEither[Coeval](name.toEither) + r <- ctx.functionTypeSignaturesByName(name) match { + case Nil => EitherT.fromEither[Coeval](Left(s"Function '$name' not found")) + case singleOverload :: Nil => resolvedFuncArguments(ctx, args).subflatMap(matchFuncOverload(name, args, _, singleOverload)) + case many => + resolvedFuncArguments(ctx, args).subflatMap { resolvedArgs => + val matchedSignatures = many + .zip(many.map(matchFuncOverload(name, args, resolvedArgs, _))) + .collect { + case (sig, result) if result.isRight => (sig, result) + } - ctx.functionTypeSignaturesByName(name) match { - case Nil => EitherT.fromEither[Coeval](Left(s"Function '$name' not found")) - case singleOverload :: Nil => resolvedArguments(args).subflatMap(matchOverload(_, singleOverload)) - case many => - resolvedArguments(args).subflatMap { resolvedArgs => - val matchedSignatures = many - .zip(many.map(matchOverload(resolvedArgs, _))) - .collect { - case (sig, result) if result.isRight => (sig, result) + matchedSignatures match { + case Nil => Left(s"Can't find a function '$name'(${resolvedArgs.map(_.tpe.typeInfo).mkString(", ")})") + case (_, oneFuncResult) :: Nil => oneFuncResult + case manyPairs => + val candidates = manyPairs.map { case (sig, _) => s"'$name'(${sig.args.mkString(", ")})" } + Left(s"Can't choose an overloaded function. Candidates: ${candidates.mkString("; ")}") } - - matchedSignatures match { - case Nil => Left(s"Can't find a function '$name'(${resolvedArgs.map(_.tpe.typeInfo).mkString(", ")})") - case (_, oneFuncResult) :: Nil => oneFuncResult - case manyPairs => - val candidates = manyPairs.map { case (sig, _) => s"'$name'(${sig.args.mkString(", ")})" } - Left(s"Can't choose an overloaded function. Candidates: ${candidates.mkString("; ")}") } - } - } + } + } yield r } - private def compileBlock(ctx: CompilerContext, block: Expressions.BLOCK): SetTypeResult[EXPR] = { - import block.let - (ctx.varDefs.get(let.name), ctx.functionDefs.get(let.name)) match { - case (Some(_), _) => EitherT.leftT[Coeval, EXPR](s"Value '${let.name}' already defined in the scope") - case (_, Some(_)) => - EitherT.leftT[Coeval, EXPR](s"Value '${let.name}' can't be defined because function with such name is predefined") - case (None, None) => - for { - exprTpe <- compile(ctx, EitherT.pure(let.value)) - _ <- EitherT - .cond[Coeval](let.types.forall(ctx.predefTypes.contains), (), s"Value '${let.name}' declared as non-existing type") - desiredUnion = if (let.types.isEmpty) exprTpe.tpe else UNION(let.types.toList.map(CASETYPEREF)) - updatedCtx = ctx.copy(varDefs = ctx.varDefs + (let.name -> desiredUnion)) - inExpr <- compile(updatedCtx, EitherT.pure(block.body)) - } yield - BLOCK( - let = LET(let.name, exprTpe), - body = inExpr, - tpe = inExpr.tpe - ) - } - } + private def compileBlock(ctx: CompilerContext, block: Expressions.BLOCK): SetTypeResult[EXPR] = + for { + letName <- EitherT.fromEither[Coeval](block.let.name.toEither) + r <- (ctx.varDefs.get(letName), ctx.functionDefs.get(letName)) match { + case (Some(_), _) => EitherT.leftT[Coeval, EXPR](s"Value '$letName' already defined in the scope") + case (_, Some(_)) => + EitherT.leftT[Coeval, EXPR](s"Value '$letName' can't be defined because function with such name is predefined") + case (None, None) => + import block.let + for { + exprTpe <- compile(ctx, EitherT.pure(let.value)) + letTypes <- EitherT.fromEither[Coeval](let.types.map(_.toEither).toList.sequence[CompilationResult, String]) + _ <- EitherT.cond[Coeval](letTypes.forall(ctx.predefTypes.contains), (), s"Value '$letName' declared as non-existing type") + desiredUnion = if (let.types.isEmpty) exprTpe.tpe else UNION(letTypes.map(CASETYPEREF)) + updatedCtx = ctx.copy(varDefs = ctx.varDefs + (letName -> desiredUnion)) + inExpr <- compile(updatedCtx, EitherT.pure(block.body)) + } yield + BLOCK( + let = LET(letName, exprTpe), + body = inExpr, + tpe = inExpr.tpe + ) + } + } yield r private def compileRef(ctx: CompilerContext, ref: Expressions.REF): SetTypeResult[EXPR] = EitherT.fromEither { - ctx.varDefs - .get(ref.key) - .map { tpe => - REF(key = ref.key, tpe = tpe) - } - .toRight(s"A definition of '${ref.key}' is not found") + ref.key.toEither.flatMap { key => + ctx.varDefs + .get(key) + .map(REF(key, _)) + .toRight(s"A definition of '$key' is not found") + } } private def compileMatch(ctx: CompilerContext, m: Expressions.MATCH): SetTypeResult[EXPR] = { @@ -204,13 +191,13 @@ object CompilerV1 { for { typedExpr <- compile(ctx, EitherT.pure(expr)) - possibleExpressionTypes <- typedExpr.tpe match { - case u: UNION => EitherT.fromEither[Coeval](Right(u)) - case _ => EitherT.fromEither[Coeval](Left("Only union type can be matched")) - } - matchingTypes = cases.flatMap(_.types) - matchedTypes = UNION(matchingTypes.toList.map(CASETYPEREF)) - lastEmpty = cases.last.types.isEmpty + possibleExpressionTypes <- EitherT.fromEither[Coeval](typedExpr.tpe match { + case u: UNION => Right(u) + case _ => Left("Only union type can be matched") + }) + matchingTypes <- EitherT.fromEither[Coeval](cases.flatMap(_.types).map(_.toEither).toList.sequence[CompilationResult, String]) + matchedTypes = UNION(matchingTypes.map(CASETYPEREF)) + lastEmpty = cases.last.types.isEmpty _ <- EitherT.cond[Coeval]( lastEmpty && (possibleExpressionTypes >= matchedTypes) || (possibleExpressionTypes equivalent matchedTypes), (), @@ -226,17 +213,52 @@ object CompilerV1 { other) } val blockWithNewVar = mc.newVarName match { - case Some(nweVal) => Expressions.BLOCK(Expressions.LET(nweVal, refTmp, mc.types), mc.expr) + case Some(newVal) => Expressions.BLOCK(Expressions.LET(newVal, refTmp, mc.types), mc.expr) case None => mc.expr } if (typeSwarma == Expressions.FALSE) blockWithNewVar else Expressions.IF(typeSwarma, blockWithNewVar, further) } - compiled <- compileBlock(updatedCtx, Expressions.BLOCK(Expressions.LET(rootMatchTmpArg, expr), ifBasedCases)) + compiled <- compileBlock(updatedCtx, Expressions.BLOCK(Expressions.LET(rootMatchTmpArg, expr, Seq.empty), ifBasedCases)) } yield compiled } + private def resolvedFuncArguments(ctx: CompilerContext, args: List[Expressions.EXPR]): ResolvedArgsResult = { + import cats.instances.list._ + val r: List[SetTypeResult[EXPR]] = args.map(arg => compile(ctx, EitherT.pure(arg)))(collection.breakOut) + r.sequence[SetTypeResult, EXPR] + } + + private def matchFuncOverload(funcName: String, + funcArgs: List[Expressions.EXPR], + resolvedArgs: List[EXPR], + f: FunctionTypeSignature): Either[String, EXPR] = { + val argTypes = f.args + val resultType = f.result + if (funcArgs.lengthCompare(argTypes.size) != 0) + Left(s"Function '$funcName' requires ${argTypes.size} arguments, but ${funcArgs.size} are provided") + else { + val typedExpressionArgumentsAndTypedPlaceholders: List[(EXPR, TYPEPLACEHOLDER)] = resolvedArgs.zip(argTypes) + + val typePairs = typedExpressionArgumentsAndTypedPlaceholders.map { case (typedExpr, tph) => (typedExpr.tpe, tph) } + for { + resolvedTypeParams <- TypeInferrer(typePairs) + resolvedResultType <- TypeInferrer.inferResultType(resultType, resolvedTypeParams) + } yield + FUNCTION_CALL( + FunctionHeader(funcName, f.args.map(FunctionHeaderType.fromTypePlaceholder)), + typedExpressionArgumentsAndTypedPlaceholders.map(_._1), + resolvedResultType + ) + } + } + + private def handlePart[T](part: PART[T])(f: T => EXPR): SetTypeResult[EXPR] = part match { + case PART.VALID(x) => EitherT.pure(f(x)) + case PART.INVALID(x, message) => EitherT.leftT[Coeval, EXPR](s"$message: $x") + } + def apply(c: CompilerContext, expr: Expressions.EXPR): CompilationResult[EXPR] = { def result = compile(c, EitherT.pure(expr)).value().left.map { e => s"Typecheck failed: $e" diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/Terms.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/Terms.scala index 4aaa2c29f58..a400e9c0652 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/Terms.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/Terms.scala @@ -38,7 +38,6 @@ object Terms { case class CASETYPEREF(name: String) extends AUTO_TAGGED_TYPE[CaseObj] case class UNION(l: List[CASETYPEREF]) extends AUTO_TAGGED_TYPE[AnyObj] object UNION { - implicit class UnionExt(l1: UNION) { def equivalent(l2: UNION): Boolean = l1.l.toSet == l2.l.toSet diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala index 06591b88f3b..ea8d5079d49 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala @@ -1,49 +1,49 @@ package com.wavesplatform.lang.v1.parser -import Expressions._ +import com.wavesplatform.lang.v1.parser.Expressions._ import fastparse.all._ sealed abstract class BinaryOperation { val func: String val parser: P[Any] = P(func) def expr(op1: EXPR)(op2: EXPR): EXPR = { - BINARY_OP(op1, this, op2) + BINARY_OP(op1, this, op2) } } object BinaryOperation { - val opsByPriority = List[BinaryOperation]( + val opsByPriority: List[BinaryOperation] = List[BinaryOperation]( OR_OP, AND_OP, EQ_OP, NE_OP, - GE_OP, GT_OP, - LE_OP, + GE_OP, LT_OP, + LE_OP, SUM_OP, SUB_OP ) - def opsToFunctions(op: BinaryOperation) = op.func + def opsToFunctions(op: BinaryOperation): String = op.func - case object OR_OP extends BinaryOperation { + case object OR_OP extends BinaryOperation { val func = "||" } case object AND_OP extends BinaryOperation { val func = "&&" } - case object EQ_OP extends BinaryOperation { + case object EQ_OP extends BinaryOperation { val func = "==" } - case object NE_OP extends BinaryOperation { + case object NE_OP extends BinaryOperation { val func = "!=" } - case object GE_OP extends BinaryOperation { + case object GE_OP extends BinaryOperation { val func = ">=" } - case object GT_OP extends BinaryOperation { + case object GT_OP extends BinaryOperation { val func = ">" } case object SUM_OP extends BinaryOperation { @@ -52,18 +52,18 @@ object BinaryOperation { case object SUB_OP extends BinaryOperation { val func = "-" } - case object LE_OP extends BinaryOperation { - val func = ">=" + case object LE_OP extends BinaryOperation { + val func = ">=" override val parser = P("<=") override def expr(op1: EXPR)(op2: EXPR): EXPR = { - BINARY_OP(op2, LE_OP, op1) + BINARY_OP(op2, LE_OP, op1) } } - case object LT_OP extends BinaryOperation { - val func = ">" + case object LT_OP extends BinaryOperation { + val func = ">" override val parser = P("<") override def expr(op1: EXPR)(op2: EXPR): EXPR = { - BINARY_OP(op2, LT_OP, op1) + BINARY_OP(op2, LT_OP, op1) } } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala index b9fd1f7fd19..75bd7ff1dbe 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala @@ -4,20 +4,70 @@ import scodec.bits.ByteVector object Expressions { - case class LET(name: String, value: EXPR, types: Seq[String] = Seq.empty) + sealed trait PART[+T] + object PART { + case class VALID[T](v: T) extends PART[T] + case class INVALID(consumed: String, message: String) extends PART[Nothing] + } + + case class LET(name: PART[String], value: EXPR, types: Seq[PART[String]]) + object LET { + def apply(name: String, value: EXPR, types: Seq[String]): LET = LET(PART.VALID(name), value, types.map(PART.VALID(_))) + } + sealed trait EXPR - case class CONST_LONG(value: Long) extends EXPR - case class GETTER(ref: EXPR, field: String) extends EXPR - case class CONST_BYTEVECTOR(value: ByteVector) extends EXPR - case class CONST_STRING(value: String) extends EXPR + case class CONST_LONG(value: Long) extends EXPR + + case class GETTER(ref: EXPR, field: PART[String]) extends EXPR + object GETTER { + def apply(ref: EXPR, field: String): GETTER = GETTER(ref, PART.VALID(field)) + } + + case class CONST_BYTEVECTOR(value: PART[ByteVector]) extends EXPR + object CONST_BYTEVECTOR { + def apply(x: ByteVector): CONST_BYTEVECTOR = CONST_BYTEVECTOR(PART.VALID(x)) + } + + case class CONST_STRING(value: PART[String]) extends EXPR + object CONST_STRING { + def apply(x: String): CONST_STRING = CONST_STRING(PART.VALID(x)) + } + case class BINARY_OP(a: EXPR, kind: BinaryOperation, b: EXPR) extends EXPR case class BLOCK(let: LET, body: EXPR) extends EXPR case class IF(cond: EXPR, ifTrue: EXPR, ifFalse: EXPR) extends EXPR - case class REF(key: String) extends EXPR - case object TRUE extends EXPR - case object FALSE extends EXPR - case class FUNCTION_CALL(name: String, args: List[EXPR]) extends EXPR - case class MATCH_CASE(newVarName: Option[String], types: Seq[String], expr: EXPR) + + case class REF(key: PART[String]) extends EXPR + object REF { + def apply(key: String): REF = REF(PART.VALID(key)) + } + + case object TRUE extends EXPR + case object FALSE extends EXPR + + case class FUNCTION_CALL(name: PART[String], args: List[EXPR]) extends EXPR + object FUNCTION_CALL { + def apply(name: String, args: List[EXPR]): FUNCTION_CALL = FUNCTION_CALL(PART.VALID(name), args) + } + + case class MATCH_CASE(newVarName: Option[PART[String]], types: Seq[PART[String]], expr: EXPR) + object MATCH_CASE { + def apply(newVarName: Option[String], types: List[String], expr: EXPR): MATCH_CASE = + MATCH_CASE(newVarName.map(PART.VALID(_)), types.map(PART.VALID(_)), expr) + } + case class MATCH(expr: EXPR, cases: Seq[MATCH_CASE]) extends EXPR + case class INVALID(message: String, next: Option[EXPR] = None) extends EXPR + object INVALID { + def apply(message: String, next: EXPR): INVALID = INVALID(message, Some(next)) + } + + implicit class PartOps[T](val self: PART[T]) extends AnyVal { + def toEither: Either[String, T] = self match { + case Expressions.PART.VALID(x) => Right(x) + case Expressions.PART.INVALID(x, message) => Left(s"$message: $x") + } + } + } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala index 50a52f2ede6..17771d5ac97 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala @@ -1,8 +1,8 @@ package com.wavesplatform.lang.v1.parser -import Expressions._ -import BinaryOperation._ -import UnaryOperation._ +import com.wavesplatform.lang.v1.parser.BinaryOperation._ +import com.wavesplatform.lang.v1.parser.Expressions._ +import com.wavesplatform.lang.v1.parser.UnaryOperation._ import fastparse.{WhitespaceApi, core} import scodec.bits.ByteVector @@ -17,45 +17,138 @@ object Parser { import White._ import fastparse.noApi._ - private val Base58Chars = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" - val keywords = Set("let", "base58", "true", "false", "if", "then", "else", "match", "case") - private val lowerChar = CharIn('a' to 'z') - private val upperChar = CharIn('A' to 'Z') - private val char = lowerChar | upperChar - private val digit = CharIn('0' to '9') - private val unicodeSymbolP = P("u" ~ P(digit | char) ~ P(digit | char) ~ P(digit | char) ~ P(digit | char)) - private val escapedUnicodeSymbolP = P("\\" ~ (CharIn("\"\\bfnrt") | unicodeSymbolP)) - private val varName = (char.repX(min = 1, max = 1) ~~ (digit | char).repX()).!.filter(!keywords.contains(_)) + val keywords = Set("let", "base58", "true", "false", "if", "then", "else", "match", "case") + private val lowerChar = CharIn('a' to 'z') + private val upperChar = CharIn('A' to 'Z') + private val char = lowerChar | upperChar + private val digit = CharIn('0' to '9') + private val unicodeSymbolP = P("\\u" ~/ Pass ~~ (char | digit).repX(min = 0, max = 4)) + private val notEndOfString = CharPred(_ != '\"') + private val specialSymbols = P("\\" ~~ notEndOfString.?) + + private val escapedUnicodeSymbolP = P(NoCut(unicodeSymbolP) | specialSymbols) + private val stringP: P[EXPR] = P("\"" ~/ Pass ~~ (escapedUnicodeSymbolP | notEndOfString).!.repX ~~ "\"") + .map { xs => + var errors = Vector.empty[String] + val consumedString = new StringBuilder + + xs.foreach { x => + if (x.startsWith("\\u")) { + if (x.length == 6) { + val hexCode = x.drop(2) + try { + val int = Integer.parseInt(hexCode, 16) + val unicodeSymbol = new String(Character.toChars(int)) + consumedString.append(unicodeSymbol) + } catch { + case _: NumberFormatException => + consumedString.append(x) + errors :+= s"Can't parse '$hexCode' as HEX string in '$x'" + case _: IllegalArgumentException => + consumedString.append(x) + errors :+= s"Invalid UTF-8 symbol: '$x'" + } + } else { + consumedString.append(x) + errors :+= s"Incomplete UTF-8 symbol definition: '$x'" + } + } else if (x.startsWith("\\")) { + if (x.length == 2) { + consumedString.append(x(1) match { + case 'b' => "\b" + case 'f' => "\f" + case 'n' => "\n" + case 'r' => "\r" + case 't' => "\t" + case _ => + errors :+= s"Unknown escaped symbol: '$x'" + x + }) + } else { + consumedString.append(x) + errors :+= s"Invalid escaped symbol: '$x'" + } + } else { + consumedString.append(x) + } + } + + if (errors.isEmpty) PART.VALID(consumedString.toString) + else PART.INVALID(consumedString.toString, errors.mkString(";")) + } + .map(CONST_STRING(_)) - private val expr = P(binaryOp(opsByPriority) | atom) + private val varName: P[PART[String]] = (char ~~ (digit | char).repX()).!.map { x => + if (keywords.contains(x)) PART.INVALID(x, "keywords are restricted") + else PART.VALID(x) + } + + private val invalid: P[INVALID] = P(AnyChars(1).! ~ fallBackExpr.?).map { + case (xs, next) => foldInvalid(xs, next) + } + + private def foldInvalid(xs: String, next: Option[EXPR]): INVALID = next match { + case Some(INVALID(nextXs, nextNext)) => foldInvalid(xs + nextXs, nextNext) + case x => INVALID(xs, x) + } - private val numberP: P[CONST_LONG] = P(CharIn("+-").rep(max = 1) ~ digit.repX(min = 1)).!.map(t => CONST_LONG(t.toLong)) + private val numberP: P[CONST_LONG] = P(CharIn("+-").? ~ digit.repX(min = 1)).!.map(t => CONST_LONG(t.toLong)) private val trueP: P[TRUE.type] = P("true").map(_ => TRUE) private val falseP: P[FALSE.type] = P("false").map(_ => FALSE) - private val bracesP: P[EXPR] = P("(" ~ expr ~ ")") - private val curlyBracesP: P[EXPR] = P("{" ~ expr ~ "}") - private val letP: P[LET] = P("let" ~ varName ~ "=" ~ expr).map { case ((x, y)) => LET(x, y) } - private val refP: P[REF] = P(varName).map(x => REF(x)) - private val ifP: P[IF] = P("if" ~ bracesP ~ "then" ~ expr ~ "else" ~ expr).map { case (x, y, z) => IF(x, y, z) } + private val bracesP: P[EXPR] = P("(" ~ fallBackExpr ~ ")") + private val curlyBracesP: P[EXPR] = P("{" ~ fallBackExpr ~ "}") + private val letP: P[LET] = P("let" ~ varName ~ "=" ~ fallBackExpr).map(Function.tupled(LET(_, _, Seq.empty))) + private val refP: P[REF] = P(varName).map(REF(_)) + private val ifP: P[IF] = P("if" ~ bracesP ~ "then" ~ fallBackExpr ~ "else" ~ fallBackExpr).map { case (x, y, z) => IF(x, y, z) } - private val functionCallArgs: P[Seq[EXPR]] = expr.rep(sep = ",") + private val functionCallArgs: P[Seq[EXPR]] = fallBackExpr.rep(sep = ",") private val extractableAtom: P[EXPR] = P(curlyBracesP | bracesP | refP) private abstract class Accessor - private case class Getter(name: String) extends Accessor - private case class Args(args: Seq[EXPR]) extends Accessor - private case class ListIndex(index: EXPR) extends Accessor - - private val typesP: P[Seq[String]] = varName.rep(min = 1, sep = "|") - private val matchCaseP: P[MATCH_CASE] = P("case" ~ (varName.map(Some.apply) | P("_").map(_ => None)) ~ (P(":" ~ typesP) | P("").map(_ => List())) ~ "=>" ~ expr).map { case (v, types, e) => MATCH_CASE(v, types, e) } - private lazy val matchP: P[MATCH] = P("match" ~ expr ~ "{" ~ matchCaseP.rep(min = 1) ~ "}").map { case (e, cases) => MATCH(e, cases.toList) } + private case class Getter(name: PART[String]) extends Accessor + private case class Args(args: Seq[EXPR]) extends Accessor + private case class ListIndex(index: EXPR) extends Accessor + + private val matchCaseP: P[MATCH_CASE] = { + val restMatchCaseInvalidP: P[String] = P((!"=>" ~~ AnyChars(1).!).repX.map(_.mkString)) + val varDefP: P[Option[PART[String]]] = varName.map(Some(_)) | "_".!.map(_ => None) + val typesP: P[Seq[PART[String]]] = varName.rep(min = 1, sep = "|") + val typesDefP = ( + ":" ~ + (typesP | restMatchCaseInvalidP.map(x => Seq(PART.INVALID(x, "the type for variable should be specified: `case varName: Type => expr`")))) + ).?.map(_.getOrElse(List.empty)) + + P( + "case" ~/ ( + (varDefP ~ typesDefP) | + restMatchCaseInvalidP.map { x => + ( + Some(PART.INVALID(x, "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), + Seq.empty[PART[String]] + ) + } + ) ~ "=>" ~/ baseExpr.? + ).map { + case (v, types, e) => + MATCH_CASE( + newVarName = v, + types = types, + expr = e.getOrElse(INVALID("expected expression")) + ) + } + } + private lazy val matchP: P[EXPR] = P("match" ~/ fallBackExpr ~ "{" ~ NoCut(matchCaseP).rep ~ "}") + .map { + case (_, Nil) => INVALID("pattern matching requires case branches") + case (e, cases) => MATCH(e, cases.toList) + } - private val accessP: P[Accessor] = P(("." ~~ varName).map(Getter.apply) | ("(" ~/ functionCallArgs.map(Args.apply) ~ ")")) | ("[" ~/ expr.map( - ListIndex.apply) ~ "]") + private val accessP + : P[Accessor] = P(("." ~~ varName).map(Getter) | ("(" ~/ functionCallArgs.map(Args) ~ ")")) | ("[" ~/ fallBackExpr.map(ListIndex) ~ "]") - private val maybeAccessP: P[EXPR] = P(extractableAtom ~~ accessP.rep).map { + private val maybeAccessP: P[EXPR] = P(extractableAtom ~~ NoCut(accessP).rep).map { case (e, f) => f.foldLeft(e) { (e, a) => a match { @@ -63,45 +156,47 @@ object Parser { case Args(args) => e match { case REF(functionName) => FUNCTION_CALL(functionName, args.toList) - case _ => ??? + case _ => FUNCTION_CALL(PART.INVALID("", s"$e is not a function name"), args.toList) } case ListIndex(index) => FUNCTION_CALL("getElement", List(e, index)) } } } - private val byteVectorP: P[CONST_BYTEVECTOR] = - P("base58'" ~~ CharsWhileIn(Base58Chars, 0).! ~~ "'") - .map { x => - if (x.isEmpty) Right(Array.emptyByteArray) else Global.base58Decode(x) - } - .flatMap { - case Left(e) => Fail.opaque(e) - case Right(xs) => PassWith(CONST_BYTEVECTOR(ByteVector(xs))) + private val byteVectorP: P[EXPR] = + P("base58'" ~/ Pass ~~ CharPred(_ != '\'').repX.! ~~ "'") + .map { xs => + val decoded = if (xs.isEmpty) Right(Array.emptyByteArray) else Global.base58Decode(xs) + decoded match { + case Left(_) => CONST_BYTEVECTOR(PART.INVALID(xs, "Can't parse Base58 string")) + case Right(r) => CONST_BYTEVECTOR(PART.VALID(ByteVector(r))) + } } - private val stringP: P[CONST_STRING] = - P("\"" ~~ (CharsWhile(!"\"\\".contains(_: Char)) | escapedUnicodeSymbolP).rep.! ~~ "\"").map(CONST_STRING) - - private val block: P[EXPR] = P(letP ~ expr).map(Function.tupled(BLOCK.apply)) + private val block: P[EXPR] = P(letP ~ fallBackExpr).map(Function.tupled(BLOCK)) - private val atom = P(ifP | matchP | byteVectorP | stringP | numberP | trueP | falseP | block | maybeAccessP) + private val baseAtom = P(ifP | NoCut(matchP) | byteVectorP | stringP | numberP | trueP | falseP | block | maybeAccessP) + private lazy val baseExpr = P(binaryOp(baseAtom, opsByPriority) | baseAtom) - def unaryOp(ops: Seq[(P[Any], EXPR => EXPR)]): P[EXPR] = { - ops.foldRight(atom) { - (op, acc) => (op._1.map(_ => ()) ~ P(unaryOp(ops))).map(op._2) | acc - } + private lazy val fallBackExpr = { + val fallBackAtom = P(baseAtom | invalid) + P(binaryOp(fallBackAtom, opsByPriority) | fallBackAtom) } - private def binaryOp(rest: List[(BinaryOperation)]): P[EXPR] = rest match { - case Nil => unaryOp(unaryOps) + private def binaryOp(atom: P[EXPR], rest: List[BinaryOperation]): P[EXPR] = rest match { + case Nil => unaryOp(atom, unaryOps) case kind :: restOps => - val operand = binaryOp(restOps) - P(operand ~ (kind.parser.!.map(_ => kind) ~ operand).rep()).map { + val operand = binaryOp(atom, restOps) + P(operand ~ (kind.parser.!.map(_ => kind) ~/ operand).rep()).map { case (left: EXPR, r: Seq[(BinaryOperation, EXPR)]) => r.foldLeft(left) { case (acc, (currKind, currOperand)) => currKind.expr(acc)(currOperand) } } } - def apply(str: String): core.Parsed[EXPR, Char, String] = P(Start ~ expr ~ End).parse(str) + def unaryOp(atom: P[EXPR], ops: Seq[(P[Any], EXPR => EXPR)]): P[EXPR] = ops.foldRight(atom) { + case ((parser, transformer), acc) => + (parser.map(_ => ()) ~ P(unaryOp(atom, ops))).map(transformer) | acc + } + + def apply(str: String): core.Parsed[Seq[EXPR], Char, String] = P(Start ~ fallBackExpr.rep(min = 1) ~ End).parse(str) } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala index 31f346e4b70..2e25f482b26 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala @@ -1,11 +1,15 @@ package com.wavesplatform.lang.v1.parser -import Expressions._ +import com.wavesplatform.lang.v1.parser.Expressions._ import fastparse.all._ object UnaryOperation { - val unaryOps = List( - P("-" ~ !CharIn('0' to '9')) -> {e: EXPR => FUNCTION_CALL("-", List(e))}, - P("!") -> {e: EXPR => FUNCTION_CALL("!", List(e))} - ) + val unaryOps = List( + P("-" ~ !CharIn('0' to '9')) -> { e: EXPR => + FUNCTION_CALL("-", List(e)) + }, + P("!") -> { e: EXPR => + FUNCTION_CALL("!", List(e)) + } + ) } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index ea70c055e97..424f0b88bf6 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -1,28 +1,33 @@ package com.wavesplatform.lang.v1.testing -import com.wavesplatform.lang.v1.parser.Parser.keywords import com.wavesplatform.lang.v1.parser.BinaryOperation -import com.wavesplatform.lang.v1.parser.Expressions._ import com.wavesplatform.lang.v1.parser.BinaryOperation._ +import com.wavesplatform.lang.v1.parser.Expressions._ +import com.wavesplatform.lang.v1.parser.Parser.keywords import org.scalacheck._ +import scodec.bits.ByteVector +import scorex.crypto.encode.Base58 + +import scala.reflect.ClassTag trait ScriptGen { def CONST_LONGgen: Gen[(EXPR, Long)] = Gen.choose(Long.MinValue, Long.MaxValue).map(v => (CONST_LONG(v), v)) - def BOOLgen(gas: Int): Gen[(EXPR,Boolean)] = - if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), NE_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1)) + def BOOLgen(gas: Int): Gen[(EXPR, Boolean)] = + if (gas > 0) + Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), NE_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1)) else Gen.const((TRUE, true)) def SUMgen(gas: Int): Gen[(EXPR, Long)] = for { (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - } yield - if((BigInt(v1) + BigInt(v2)).isValidLong) { - (BINARY_OP(i1, SUM_OP, i2), (v1 + v2)) + } yield + if ((BigInt(v1) + BigInt(v2)).isValidLong) { + (BINARY_OP(i1, SUM_OP, i2), v1 + v2) } else { - (BINARY_OP(i1, SUB_OP, i2), (v1 - v2)) + (BINARY_OP(i1, SUB_OP, i2), v1 - v2) } def SUBgen(gas: Int): Gen[(EXPR, Long)] = @@ -30,85 +35,96 @@ trait ScriptGen { (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) } yield - if((BigInt(v1) - BigInt(v2)).isValidLong) { - (BINARY_OP(i1, SUB_OP, i2), (v1 - v2)) + if ((BigInt(v1) - BigInt(v2)).isValidLong) { + (BINARY_OP(i1, SUB_OP, i2), v1 - v2) } else { - (BINARY_OP(i1, SUM_OP, i2), (v1 + v2)) + (BINARY_OP(i1, SUM_OP, i2), v1 + v2) } - def INTGen(gas: Int): Gen[(EXPR, Long)] = if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), SUBgen(gas - 1), IF_INTgen(gas - 1), INTGen(gas-1).filter(v => (-BigInt(v._2)).isValidLong).map(e => (FUNCTION_CALL("-",List(e._1)), -e._2))) else CONST_LONGgen + def INTGen(gas: Int): Gen[(EXPR, Long)] = + if (gas > 0) + Gen.oneOf( + CONST_LONGgen, + SUMgen(gas - 1), + SUBgen(gas - 1), + IF_INTgen(gas - 1), + INTGen(gas - 1).filter(v => (-BigInt(v._2)).isValidLong).map(e => (FUNCTION_CALL("-", List(e._1)), -e._2)) + ) + else CONST_LONGgen def GEgen(gas: Int): Gen[(EXPR, Boolean)] = for { - dir <- Gen.oneOf(true, false) + dir <- Gen.oneOf(true, false) (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - } yield if(dir) { - (BINARY_OP(i1, GE_OP, i2), (v1 >= v2)) + } yield + if (dir) { + (BINARY_OP(i1, GE_OP, i2), v1 >= v2) } else { - (BINARY_OP(i2, LE_OP, i1), (v1 <= v2)) - } + (BINARY_OP(i2, LE_OP, i1), v1 <= v2) + } def GTgen(gas: Int): Gen[(EXPR, Boolean)] = for { - dir <- Gen.oneOf(true, false) + dir <- Gen.oneOf(true, false) (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - } yield if(dir) { - (BINARY_OP(i1, GT_OP, i2), (v1 > v2)) + } yield + if (dir) { + (BINARY_OP(i1, GT_OP, i2), v1 > v2) } else { - (BINARY_OP(i2, LT_OP, i1), (v1 < v2)) + (BINARY_OP(i2, LT_OP, i1), v1 < v2) } def EQ_INTgen(gas: Int): Gen[(EXPR, Boolean)] = for { (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - } yield (BINARY_OP(i1, EQ_OP, i2), (v1 == v2)) + } yield (BINARY_OP(i1, EQ_OP, i2), v1 == v2) def NE_INTgen(gas: Int): Gen[(EXPR, Boolean)] = for { (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - } yield (BINARY_OP(i1, NE_OP, i2), (v1 != v2)) + } yield (BINARY_OP(i1, NE_OP, i2), v1 != v2) def ANDgen(gas: Int): Gen[(EXPR, Boolean)] = for { (i1, v1) <- BOOLgen((gas - 2) / 2) (i2, v2) <- BOOLgen((gas - 2) / 2) - } yield (BINARY_OP(i1, AND_OP, i2), (v1 && v2)) + } yield (BINARY_OP(i1, AND_OP, i2), v1 && v2) def ORgen(gas: Int): Gen[(EXPR, Boolean)] = for { (i1, v1) <- BOOLgen((gas - 2) / 2) (i2, v2) <- BOOLgen((gas - 2) / 2) - } yield (BINARY_OP(i1, OR_OP, i2), (v1 || v2)) + } yield (BINARY_OP(i1, OR_OP, i2), v1 || v2) def IF_BOOLgen(gas: Int): Gen[(EXPR, Boolean)] = for { (cnd, vcnd) <- BOOLgen((gas - 3) / 3) - (t, vt) <- BOOLgen((gas - 3) / 3) - (f, vf) <- BOOLgen((gas - 3) / 3) - } yield (IF(cnd, t, f), if(vcnd) { vt } else { vf }) + (t, vt) <- BOOLgen((gas - 3) / 3) + (f, vf) <- BOOLgen((gas - 3) / 3) + } yield (IF(cnd, t, f), if (vcnd) vt else vf) def IF_INTgen(gas: Int): Gen[(EXPR, Long)] = for { (cnd, vcnd) <- BOOLgen((gas - 3) / 3) - (t, vt) <- INTGen((gas - 3) / 3) - (f, vf) <- INTGen((gas - 3) / 3) - } yield (IF(cnd, t, f), if(vcnd) { vt } else { vf }) + (t, vt) <- INTGen((gas - 3) / 3) + (f, vf) <- INTGen((gas - 3) / 3) + } yield (IF(cnd, t, f), if (vcnd) vt else vf) def STRgen: Gen[EXPR] = - Gen.identifier.map(CONST_STRING) + Gen.identifier.map(PART.VALID[String]).map(CONST_STRING(_)) def LETgen(gas: Int): Gen[LET] = for { - name <- Gen.identifier + name <- Gen.identifier (value, _) <- BOOLgen((gas - 3) / 3) - } yield LET(name, value) + } yield LET(PART.VALID(name), value, Seq.empty) def REFgen: Gen[EXPR] = - Gen.identifier.filter(!keywords(_)).map(REF) + Gen.identifier.filter(!keywords(_)).map(PART.VALID[String]).map(REF(_)) def BLOCKgen(gas: Int): Gen[EXPR] = for { @@ -130,14 +146,22 @@ trait ScriptGen { post <- whitespaces } yield s" $expr " //pred + expr + post + private def toString[T](part: PART[T])(implicit ct: ClassTag[T]): String = part match { + case PART.VALID(x: String) => x + case PART.VALID(xs: ByteVector) => Base58.encode(xs.toArray) + case PART.INVALID(consumed, _) => consumed + case _ => throw new RuntimeException(s"Can't stringify $part") + } + def toString(expr: EXPR): Gen[String] = expr match { - case CONST_LONG(x) => withWhitespaces(s"$x") - case REF(x) => withWhitespaces(s"$x") - case CONST_STRING(x) => withWhitespaces(s"""\"$x\"""") - case TRUE => withWhitespaces("true") - case FALSE => withWhitespaces("false") - case FUNCTION_CALL("-", List(CONST_LONG(v))) if (v>=0) => s"-($v)" - case FUNCTION_CALL(op, List(e)) => toString(e).map(e => s"$op$e") + case CONST_LONG(x) => withWhitespaces(s"$x") + case REF(x) => withWhitespaces(toString(x)) + case CONST_STRING(x) => withWhitespaces(s"""\"${toString(x)}\"""") + case CONST_BYTEVECTOR(x) => withWhitespaces(s"""base58'${toString(x)}'""") + case TRUE => withWhitespaces("true") + case FALSE => withWhitespaces("false") + case FUNCTION_CALL(PART.VALID("-"), List(CONST_LONG(v))) if v >= 0 => s"-($v)" + case FUNCTION_CALL(op, List(e)) => toString(e).map(e => s"${toString(op)}$e") case BINARY_OP(x, LE_OP, y) => for { arg2 <- toString(x) @@ -163,16 +187,27 @@ trait ScriptGen { for { v <- toString(let.value) b <- toString(body) - } yield s"let ${let.name} = $v $b\n" + } yield s"let ${toString(let.name)} = $v $b\n" case _ => ??? } } trait ScriptGenParser extends ScriptGen { override def BOOLgen(gas: Int): Gen[(EXPR, Boolean)] = { - if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1), REFgen.map(r => (r, false)), BOOLgen(gas-1).map(e => (FUNCTION_CALL("!", List(e._1)), !e._2))) + if (gas > 0) + Gen.oneOf( + GEgen(gas - 1), + GTgen(gas - 1), + EQ_INTgen(gas - 1), + ANDgen(gas - 1), + ORgen(gas - 1), + IF_BOOLgen(gas - 1), + REFgen.map(r => (r, false)), + BOOLgen(gas - 1).map(e => (FUNCTION_CALL("!", List(e._1)), !e._2)) + ) else Gen.const((TRUE, true)) } - override def INTGen(gas: Int): Gen[(EXPR, Long)] = if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), IF_INTgen(gas - 1), REFgen.map(r => (r, 0L))) else CONST_LONGgen + override def INTGen(gas: Int): Gen[(EXPR, Long)] = + if (gas > 0) Gen.oneOf(CONST_LONGgen, SUMgen(gas - 1), IF_INTgen(gas - 1), REFgen.map(r => (r, 0L))) else CONST_LONGgen } diff --git a/src/test/scala/com/wavesplatform/state/diffs/AssetTransactionsDiffTest.scala b/src/test/scala/com/wavesplatform/state/diffs/AssetTransactionsDiffTest.scala index 7c939a32848..4b939cca275 100644 --- a/src/test/scala/com/wavesplatform/state/diffs/AssetTransactionsDiffTest.scala +++ b/src/test/scala/com/wavesplatform/state/diffs/AssetTransactionsDiffTest.scala @@ -218,7 +218,8 @@ class AssetTransactionsDiffTest extends PropSpec with PropertyChecks with Matche private def createScript(code: String) = { val Parsed.Success(expr, _) = Parser(code).get - ScriptV1(CompilerV1(dummyTypeCheckerContext, expr).explicitGet()).explicitGet() + assert(expr.size == 1) + ScriptV1(CompilerV1(dummyTypeCheckerContext, expr.head).explicitGet()).explicitGet() } def genesisIssueTransferReissue(code: String): Gen[(Seq[GenesisTransaction], IssueTransactionV2, TransferTransactionV1, ReissueTransactionV1)] = diff --git a/src/test/scala/com/wavesplatform/state/diffs/smart/performance/SigVerifyPerformanceTest.scala b/src/test/scala/com/wavesplatform/state/diffs/smart/performance/SigVerifyPerformanceTest.scala index fde9a8fd07f..d704225e528 100644 --- a/src/test/scala/com/wavesplatform/state/diffs/smart/performance/SigVerifyPerformanceTest.scala +++ b/src/test/scala/com/wavesplatform/state/diffs/smart/performance/SigVerifyPerformanceTest.scala @@ -53,7 +53,8 @@ class SigVerifyPerformanceTest extends PropSpec with PropertyChecks with Matcher ignore("parallel native signature verification vs sequential scripted signature verification") { val textScript = "sigVerify(tx.bodyBytes,tx.proofs[0],tx.senderPk)" val untypedScript = Parser(textScript).get.value - val typedScript = CompilerV1(dummyTypeCheckerContext, untypedScript).explicitGet() + assert(untypedScript.size == 1) + val typedScript = CompilerV1(dummyTypeCheckerContext, untypedScript.head).explicitGet() forAll(differentTransfers(typedScript)) { case (gen, setScript, transfers, scriptTransfers) => diff --git a/src/test/scala/com/wavesplatform/state/diffs/smart/predef/package.scala b/src/test/scala/com/wavesplatform/state/diffs/smart/predef/package.scala index fecae6326b1..96ba170bccb 100644 --- a/src/test/scala/com/wavesplatform/state/diffs/smart/predef/package.scala +++ b/src/test/scala/com/wavesplatform/state/diffs/smart/predef/package.scala @@ -13,7 +13,8 @@ package object predef { val networkByte: Byte = 'u' def runScript[T: TypeInfo](script: String, tx: Transaction = null): Either[String, T] = { val Success(expr, _) = Parser(script) - val Right(typedExpr) = CompilerV1(dummyTypeCheckerContext, expr) + assert(expr.size == 1) + val Right(typedExpr) = CompilerV1(dummyTypeCheckerContext, expr.head) EvaluatorV1[T](BlockchainContext.build(networkByte, Coeval(tx), Coeval(???), null), typedExpr).left.map(_._3) } } diff --git a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/AddressFromRecipientScenarioTest.scala b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/AddressFromRecipientScenarioTest.scala index ede061113e0..9df8ea46ab4 100644 --- a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/AddressFromRecipientScenarioTest.scala +++ b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/AddressFromRecipientScenarioTest.scala @@ -39,7 +39,8 @@ class AddressFromRecipientScenarioTest extends PropSpec with PropertyChecks with BlockchainContext.build(AddressScheme.current.chainId, Coeval.evalOnce(tx), Coeval.evalOnce(blockchain.height), blockchain) val Parsed.Success(expr, _) = Parser("addressFromRecipient(tx.recipient)") - val Right(typedExpr) = CompilerV1(CompilerContext.fromEvaluationContext(context), expr) + assert(expr.size == 1) + val Right(typedExpr) = CompilerV1(CompilerContext.fromEvaluationContext(context), expr.head) EvaluatorV1[Obj](context, typedExpr).left.map(_._3) } diff --git a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/HackatonScenartioTest.scala b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/HackatonScenartioTest.scala index d0c88ec9945..d6aa6c708c1 100644 --- a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/HackatonScenartioTest.scala +++ b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/HackatonScenartioTest.scala @@ -4,9 +4,9 @@ import java.nio.charset.StandardCharsets import com.wavesplatform.lang.TypeInfo import com.wavesplatform.lang.TypeInfo._ -import com.wavesplatform.lang.v1.parser.Parser import com.wavesplatform.lang.v1.compiler.CompilerV1 import com.wavesplatform.lang.v1.evaluator.EvaluatorV1 +import com.wavesplatform.lang.v1.parser.Parser import com.wavesplatform.state._ import com.wavesplatform.state.diffs._ import com.wavesplatform.state.diffs.smart._ @@ -55,8 +55,13 @@ class HackatonScenartioTest extends PropSpec with PropertyChecks with Matchers w | """.stripMargin - untypedScript = Parser(assetScript).get.value - typedScript = ScriptV1(CompilerV1(dummyTypeCheckerContext, untypedScript).explicitGet()).explicitGet() + untypedScript = { + val r = Parser(assetScript).get.value + assert(r.size == 1) + r.head + } + + typedScript = ScriptV1(CompilerV1(dummyTypeCheckerContext, untypedScript).explicitGet()).explicitGet() issueTransaction = IssueTransactionV2 .selfSigned( @@ -106,7 +111,8 @@ class HackatonScenartioTest extends PropSpec with PropertyChecks with Matchers w private def eval[T: TypeInfo](code: String) = { val untyped = Parser(code).get.value - val typed = CompilerV1(dummyTypeCheckerContext, untyped) + assert(untyped.size == 1) + val typed = CompilerV1(dummyTypeCheckerContext, untyped.head) typed.flatMap(EvaluatorV1[T](dummyContext, _)) } diff --git a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/LazyFieldAccessTest.scala b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/LazyFieldAccessTest.scala index 504a3d5e6d4..fef3b201563 100644 --- a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/LazyFieldAccessTest.scala +++ b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/LazyFieldAccessTest.scala @@ -21,7 +21,8 @@ class LazyFieldAccessTest extends PropSpec with PropertyChecks with Matchers wit private def preconditionsTransferAndLease( code: String): Gen[(GenesisTransaction, SetScriptTransaction, LeaseTransaction, TransferTransactionV1)] = { val untyped = Parser(code).get.value - val typed = CompilerV1(dummyTypeCheckerContext, untyped).explicitGet() + assert(untyped.size == 1) + val typed = CompilerV1(dummyTypeCheckerContext, untyped.head).explicitGet() preconditionsTransferAndLease(typed) } diff --git a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/MultiSig2of3Test.scala b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/MultiSig2of3Test.scala index 61a08bd4670..edf5b56d8dd 100644 --- a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/MultiSig2of3Test.scala +++ b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/MultiSig2of3Test.scala @@ -37,7 +37,8 @@ class MultiSig2of3Test extends PropSpec with PropertyChecks with Matchers with T | """.stripMargin val untyped = Parser(script).get.value - CompilerV1(dummyTypeCheckerContext, untyped).explicitGet() + assert(untyped.size == 1) + CompilerV1(dummyTypeCheckerContext, untyped.head).explicitGet() } val preconditionsAndTransfer: Gen[(GenesisTransaction, SetScriptTransaction, TransferTransactionV2, Seq[ByteStr])] = for { @@ -67,7 +68,7 @@ class MultiSig2of3Test extends PropSpec with PropertyChecks with Matchers with T property("2 of 3 multisig") { forAll(preconditionsAndTransfer) { - case ((genesis, script, transfer, sigs)) => + case (genesis, script, transfer, sigs) => val validProofs = Seq( transfer.copy(proofs = Proofs.create(Seq(sigs(0), sigs(1))).explicitGet()), transfer.copy(proofs = Proofs.create(Seq(ByteStr.empty, sigs(1), sigs(2))).explicitGet()) diff --git a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OnlyTransferIsAllowedTest.scala b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OnlyTransferIsAllowedTest.scala index 649fcbc7bbe..1c49282b8fe 100644 --- a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OnlyTransferIsAllowedTest.scala +++ b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OnlyTransferIsAllowedTest.scala @@ -23,11 +23,12 @@ class OnlyTransferIsAllowedTest extends PropSpec with PropertyChecks with Matche | else false | """.stripMargin - val untyped = Parser(scriptText).get.value - val transferAllowed = CompilerV1(dummyTypeCheckerContext, untyped).explicitGet() + val untyped = Parser(scriptText).get.value + assert(untyped.size == 1) + val transferAllowed = CompilerV1(dummyTypeCheckerContext, untyped.head).explicitGet() forAll(preconditionsTransferAndLease(transferAllowed)) { - case ((genesis, script, lease, transfer)) => + case (genesis, script, lease, transfer) => assertDiffAndState(Seq(TestBlock.create(Seq(genesis, script))), TestBlock.create(Seq(transfer)), smartEnabledFS) { case _ => () } assertDiffEi(Seq(TestBlock.create(Seq(genesis, script))), TestBlock.create(Seq(lease)), smartEnabledFS)(totalDiffEi => totalDiffEi should produce("TransactionNotAllowedByScript")) diff --git a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OracleDataTest.scala b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OracleDataTest.scala index 588dfe0e696..afa59ca77b1 100644 --- a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OracleDataTest.scala +++ b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OracleDataTest.scala @@ -30,7 +30,7 @@ class OracleDataTest extends PropSpec with PropertyChecks with Matchers with Tra bin <- binaryEntryGen(500, dataAsciiKeyGen).filter(e => e.key != long.key && e.key != bool.key) str <- stringEntryGen(500, dataAsciiKeyGen).filter(e => e.key != long.key && e.key != bool.key && e.key != bin.key) dataTransaction <- dataTransactionGenP(oracle, List(long, bool, bin, str)) - allFieldsRequiredScript = s""" + allFieldsRequiredScript = s""" | | let oracle = extract(addressFromString("${oracle.address}")) | let long = extract(getLong(oracle,"${long.key}")) == ${long.value} @@ -42,16 +42,19 @@ class OracleDataTest extends PropSpec with PropertyChecks with Matchers with Tra | | """.stripMargin - untypedAllFieldsRequiredScript = Parser(allFieldsRequiredScript).get.value - typedAllFieldsRequiredScript = CompilerV1(dummyTypeCheckerContext, untypedAllFieldsRequiredScript).explicitGet() - setScript <- selfSignedSetScriptTransactionGenP(master, ScriptV1(typedAllFieldsRequiredScript).explicitGet()) + setScript <- { + val untypedAllFieldsRequiredScript = Parser(allFieldsRequiredScript).get.value + assert(untypedAllFieldsRequiredScript.size == 1) + val typedAllFieldsRequiredScript = CompilerV1(dummyTypeCheckerContext, untypedAllFieldsRequiredScript.head).explicitGet() + selfSignedSetScriptTransactionGenP(master, ScriptV1(typedAllFieldsRequiredScript).explicitGet()) + } transferFromScripted <- versionedTransferGenP(master, alice, Proofs.empty) } yield (genesis, genesis2, setScript, dataTransaction, transferFromScripted) property("simple oracle value required to transfer") { forAll(preconditions) { - case ((genesis, genesis2, setScript, dataTransaction, transferFromScripted)) => + case (genesis, genesis2, setScript, dataTransaction, transferFromScripted) => assertDiffAndState(Seq(TestBlock.create(Seq(genesis, genesis2, setScript, dataTransaction))), TestBlock.create(Seq(transferFromScripted)), smartEnabledFS) { case _ => () } From c041566831643c01022b2fe406c9c3ddc77cc1ce Mon Sep 17 00:00:00 2001 From: Sergey Nazarov Date: Wed, 23 May 2018 19:36:44 +0300 Subject: [PATCH 39/52] Fix eviction warnings --- .travis.yml | 3 +-- build.sbt | 4 +++- project/Dependencies.scala | 29 ++++++++++++++++++----------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index e2a6bcd8271..959fe87a98f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ sudo: false jdk: - oraclejdk8 scala: -- 2.12.4 +- 2.12.6 env: global: - secure: NYdeuFBQVVjN5gon2KQEDfNfyp9Bk6AZJ7V1tfLJW54y5Dd/4PHdWGWeOyhCMIqwX3UaUmWExY1KcKXpbDMLwTmahNLjiDc3e3rCqxghL5cDgqs6OHcwNGw+CwNiw3tufo5PZ9mhK/ZJh0qgOHWyETs/ifq4VyacvgYH9IZzK4YdKYdwf+T9c3Q7VO3KmzOXiHkvPvLfkc8I8cSZlT1kJFCGawRdE/0nygPd8HYfCxiEFuXOKpRz0hlm8njjZdlZpMcou8TLeH5fFSvHtELIHFvSTEj/QilSTmCXgFRVUd8sd53n3uTouwtVbdCHsXNm4Nphf0lUJnN7fTzbD2xpOZN9zS+SO/YTkIxZpSgQLwsJsWzIHI+6IOlcjeNwZHbYBfsAdv1c4GMkgFfxsWWtb4r7+ooXDTbyy1qeZgVU4caGBMwoNE+l0Y60JdGYT4hrTfJ5c+GpcPzIV3CYVOUkHmoCYXRCtxnfDFXCR0mDVfYAA14osLi3v+EKdTxHDfqP6fICIW9y7oZd//38iEQrDy+k9iCyNmvOsqvDxXZithmSP0WzeGCXQDI+s3FxB5ywt8yDvzFbF5JYY74o1tnNFG7TlSF5v8EHSHxzXBeBtnE9V1g2wxYnvW2HYk5kW6WZJgYE9LCc6ubpQ2W387PRfCJjoLNjHqNlNGkEYlkPuaw= @@ -12,7 +12,6 @@ env: script: - sbt -S-Xfatal-warnings ";clean;coverage;generator/compile;langJS/fastOptJS;benchmark/test:compile;lang/test;test;coverageReport" after_success: - - "[[ $TRAVIS_REPO_SLUG == \"wavesplatform/Scorex\" ]] && [[ $TRAVIS_BRANCH == \"master\" ]] && { sbt publish; };" - bash <(curl -s https://codecov.io/bash) # These directories are cached to S3 at the end of the build cache: diff --git a/build.sbt b/build.sbt index 33c45160066..1356d72129f 100644 --- a/build.sbt +++ b/build.sbt @@ -43,7 +43,7 @@ logBuffered := false inThisBuild( Seq( - scalaVersion := "2.12.4", + scalaVersion := "2.12.6", organization := "com.wavesplatform", crossPaths := false, scalacOptions ++= Seq("-feature", "-deprecation", "-language:higherKinds", "-language:implicitConversions", "-Ywarn-unused:-implicits", "-Xlint") @@ -180,6 +180,8 @@ lazy val lang = .withoutSuffixFor(JVMPlatform) .settings( version := "0.0.1", + // the following line forces scala version across all dependencies + scalaModuleInfo ~= (_.map(_.withOverrideScalaVersion(true))), test in assembly := {}, libraryDependencies ++= Dependencies.cats ++ diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 66acd36c8f1..40b35e2189b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,20 +5,20 @@ object Dependencies { def akkaModule(module: String) = "com.typesafe.akka" %% s"akka-$module" % "2.4.19" - def swaggerModule(module: String) = "io.swagger" % s"swagger-$module" % "1.5.16" + def swaggerModule(module: String) = ("io.swagger" % s"swagger-$module" % "1.5.16").exclude("com.google.guava", "guava") def akkaHttpModule(module: String) = "com.typesafe.akka" %% module % "10.0.9" - def nettyModule(module: String) = "io.netty" % s"netty-$module" % "4.1.22.Final" + def nettyModule(module: String) = "io.netty" % s"netty-$module" % "4.1.24.Final" def kamonModule(v: String)(module: String) = "io.kamon" %% s"kamon-$module" % v - val asyncHttpClient = "org.asynchttpclient" % "async-http-client" % "2.1.0-alpha22" + val asyncHttpClient = "org.asynchttpclient" % "async-http-client" % "2.4.7" lazy val network = Seq("handler", "buffer", "codec").map(nettyModule) ++ Seq( "org.bitlet" % "weupnp" % "0.1.4", // Solves an issue with kamon-influxdb - asyncHttpClient + asyncHttpClient.exclude("io.netty", "netty-handler") ) lazy val testKit = scalatest ++ Seq( @@ -26,20 +26,20 @@ object Dependencies { "org.scalacheck" %% "scalacheck" % "1.13.5", "org.mockito" % "mockito-all" % "1.10.19", "org.scalamock" %% "scalamock-scalatest-support" % "3.6.0", - "org.iq80.leveldb" % "leveldb" % "0.9", + ("org.iq80.leveldb" % "leveldb" % "0.9").exclude("com.google.guava", "guava"), akkaHttpModule("akka-http-testkit") ) lazy val itKit = scalatest ++ Seq( // Swagger is using Jersey 1.1, hence the shading (https://github.com/spotify/docker-client#a-note-on-shading) - "com.spotify" % "docker-client" % "8.9.0" classifier "shaded", - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-properties" % "2.8.9", - asyncHttpClient + ("com.spotify" % "docker-client" % "8.11.3").classifier("shaded").exclude("com.google.guava", "guava"), + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-properties" % "2.9.5", + asyncHttpClient.exclude("io.netty", "netty-handler") ) lazy val serialization = Seq( "com.google.guava" % "guava" % "21.0", - "com.typesafe.play" %% "play-json" % "2.6.2" + "com.typesafe.play" %% "play-json" % "2.6.9" ) lazy val akka = Seq("actor", "slf4j").map(akkaModule) @@ -81,11 +81,18 @@ object Dependencies { "io.github.amrhassan" %% "scalacheck-cats" % "0.4.0" % Test ) lazy val meta = Seq("com.chuusai" %% "shapeless" % "2.3.3") - lazy val monix = Def.setting(Seq("io.monix" %%% "monix" % "3.0.0-RC1")) + lazy val monix = Def.setting(Seq( + // exclusion and explicit dependency can likely be removed when monix 3 is released + ("io.monix" %%% "monix" % "3.0.0-RC1").exclude("org.typelevel", "cats-effect_2.12"), + "org.typelevel" %%% "cats-effect" % "0.10.1" + )) lazy val scodec = Def.setting(Seq("org.scodec" %%% "scodec-core" % "1.10.3")) lazy val fastparse = Def.setting(Seq("com.lihaoyi" %%% "fastparse" % "1.0.0", "org.bykn" %%% "fastparse-cats-core" % "0.1.0")) lazy val ficus = Seq("com.iheart" %% "ficus" % "1.4.2") - lazy val scorex = Seq(("org.scorexfoundation" %% "scrypto" % "2.0.4").exclude("org.slf4j", "slf4j-api")) + lazy val scorex = Seq("org.scorexfoundation" %% "scrypto" % "2.0.4" excludeAll( + ExclusionRule("org.slf4j", "slf4j-api"), + ExclusionRule("com.google.guava", "guava") + )) lazy val commons_net = Seq("commons-net" % "commons-net" % "3.+") lazy val scalatest = Seq("org.scalatest" %% "scalatest" % "3.0.3") lazy val scalactic = Seq("org.scalactic" %% "scalactic" % "3.0.3") From c0e76dd03666d23296c417e100b938da8367b77d Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Wed, 23 May 2018 20:09:44 +0300 Subject: [PATCH 40/52] NODE-771 Provide sponsorship info to balance API responce. --- .../scorex/api/http/assets/AssetsApiRoute.scala | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala b/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala index d09f47c62d9..c3f3f2e8a41 100755 --- a/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala +++ b/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala @@ -237,15 +237,22 @@ case class AssetsApiRoute(settings: RestAPISettings, wallet: Wallet, utx: UtxPoo (for { (assetId, balance) <- blockchain.portfolio(acc).assets if balance > 0 - assetInfo <- blockchain.assetDescription(assetId) - issueTransaction <- blockchain.transactionInfo(assetId) + assetInfo <- blockchain.assetDescription(assetId) + (_, (issueTransaction: IssueTransaction)) <- blockchain.transactionInfo(assetId) + sponsorBalance = if (assetInfo.sponsorship != 0) { + Some(blockchain.portfolio(issueTransaction.sender).balance) + } else { + None + } } yield Json.obj( "assetId" -> assetId.base58, "balance" -> balance, "reissuable" -> assetInfo.reissuable, + "sponsorship" -> assetInfo.sponsorship, + "sponsorBalance" -> sponsorBalance, "quantity" -> JsNumber(BigDecimal(assetInfo.totalVolume)), - "issueTransaction" -> issueTransaction._2.json() + "issueTransaction" -> issueTransaction.json() )).toSeq) ) }).left.map(ApiError.fromValidationError) From 1a5e5888e99c1a403412fb4a4ca45f96c875e4e1 Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Thu, 24 May 2018 14:05:28 +0300 Subject: [PATCH 41/52] Unify fields naming. --- .../main/scala/com/wavesplatform/it/api/model.scala | 7 ++++++- .../scala/scorex/api/http/assets/AssetsApiRoute.scala | 11 +++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/it/src/main/scala/com/wavesplatform/it/api/model.scala b/it/src/main/scala/com/wavesplatform/it/api/model.scala index f80516f6ee6..d6a387b05ce 100644 --- a/it/src/main/scala/com/wavesplatform/it/api/model.scala +++ b/it/src/main/scala/com/wavesplatform/it/api/model.scala @@ -37,7 +37,12 @@ object CompiledScript { implicit val compiledScriptFormat: Format[CompiledScript] = Json.format } -case class FullAssetInfo(assetId: String, balance: Long, reissuable: Boolean, quantity: Long) +case class FullAssetInfo(assetId: String, + balance: Long, + reissuable: Boolean, + minSponsoredAssetFee: Option[Long], + sponsorBalance: Option[Long], + quantity: Long) object FullAssetInfo { implicit val fullAssetInfoFormat: Format[FullAssetInfo] = Json.format } diff --git a/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala b/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala index c3f3f2e8a41..496574cc8f2 100755 --- a/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala +++ b/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala @@ -246,10 +246,13 @@ case class AssetsApiRoute(settings: RestAPISettings, wallet: Wallet, utx: UtxPoo } } yield Json.obj( - "assetId" -> assetId.base58, - "balance" -> balance, - "reissuable" -> assetInfo.reissuable, - "sponsorship" -> assetInfo.sponsorship, + "assetId" -> assetId.base58, + "balance" -> balance, + "reissuable" -> assetInfo.reissuable, + "minSponsoredAssetFee" -> (assetInfo.sponsorship match { + case 0 => JsNull + case sponsorship => JsNumber(sponsorship) + }), "sponsorBalance" -> sponsorBalance, "quantity" -> JsNumber(BigDecimal(assetInfo.totalVolume)), "issueTransaction" -> issueTransaction.json() From 1f5ce9e9689e2f9e73139bf8da90bb71e427b85c Mon Sep 17 00:00:00 2001 From: Mike Potanin Date: Thu, 24 May 2018 15:52:46 +0300 Subject: [PATCH 42/52] Test for assets sponsorhip api. --- .../it/sync/transactions/SponsorshipSuite.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/it/src/test/scala/com/wavesplatform/it/sync/transactions/SponsorshipSuite.scala b/it/src/test/scala/com/wavesplatform/it/sync/transactions/SponsorshipSuite.scala index 43e5a7b1fe9..b02b41599f0 100644 --- a/it/src/test/scala/com/wavesplatform/it/sync/transactions/SponsorshipSuite.scala +++ b/it/src/test/scala/com/wavesplatform/it/sync/transactions/SponsorshipSuite.scala @@ -49,7 +49,6 @@ class SponsorshipSuite extends FreeSpec with NodesFromDocker with Matchers with val sponsorWavesBalance = miner.accountBalances(sponsor.address)._2 val sponsorAssetTotal = 100 * Token val minerWavesBalance = miner.accountBalances(miner.address)._2 - val aliceWavesBalance = miner.accountBalances(alice.address)._2 val sponsorAssetId = sponsor @@ -78,6 +77,9 @@ class SponsorshipSuite extends FreeSpec with NodesFromDocker with Matchers with miner.assertBalances(sponsor.address, sponsorWavesBalance - 2.waves - minWavesFee) miner.assertAssetBalance(alice.address, sponsorAssetId, sponsorAssetTotal / 2) + val assetInfo = alice.assetsBalance(alice.address).balances.filter(_.assetId == sponsorAssetId).head + assetInfo.minSponsoredAssetFee shouldBe Some(Token) + assetInfo.sponsorBalance shouldBe Some(sponsor.accountBalances(sponsor.address)._2) } "invalid tx if fee less then minimal" in { @@ -121,6 +123,11 @@ class SponsorshipSuite extends FreeSpec with NodesFromDocker with Matchers with "cancel sponsorship, cannot pay fees in non sponsored assets " in { val cancelSponsorshipTxId = sponsor.cancelSponsorship(sponsor.address, sponsorAssetId, fee = 1.waves).id nodes.waitForHeightAriseAndTxPresent(cancelSponsorshipTxId) + + val assetInfo = alice.assetsBalance(alice.address).balances.filter(_.assetId == sponsorAssetId).head + assetInfo.minSponsoredAssetFee shouldBe None + assetInfo.sponsorBalance shouldBe None + assert(!cancelSponsorshipTxId.isEmpty) assertSponsorship(sponsorAssetId, 0L) assertBadRequestAndResponse( From c9b04bb338ff6d41d459d4bf8101030cc29596c8 Mon Sep 17 00:00:00 2001 From: Alexandr M Date: Thu, 24 May 2018 16:51:48 +0300 Subject: [PATCH 43/52] NODE-129: Excpetion-blocks handling reverted --- .../wavesplatform/state/appender/package.scala | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/main/scala/com/wavesplatform/state/appender/package.scala b/src/main/scala/com/wavesplatform/state/appender/package.scala index e724023ac12..c426d8bc459 100644 --- a/src/main/scala/com/wavesplatform/state/appender/package.scala +++ b/src/main/scala/com/wavesplatform/state/appender/package.scala @@ -13,6 +13,7 @@ import scorex.consensus.TransactionsOrdering import scorex.transaction.ValidationError.{BlockAppendError, BlockFromFuture, GenericError} import scorex.transaction._ import scorex.utils.{ScorexLogging, Time} +import cats.implicits._ import scala.util.{Left, Right} @@ -20,6 +21,12 @@ package object appender extends ScorexLogging { private val MaxTimeDrift: Long = 100 // millis + // Invalid blocks, that are already in blockchain + private val exceptions = List( + 812608 -> ByteStr.decodeBase58("2GNCYVy7k3kEPXzz12saMtRDeXFKr8cymVsG8Yxx3sZZ75eHj9csfXnGHuuJe7XawbcwjKdifUrV1uMq4ZNCWPf1").get, + 813207 -> ByteStr.decodeBase58("5uZoDnRKeWZV9Thu2nvJVZ5dBvPB7k2gvpzFD618FMXCbBVBMN2rRyvKBZBhAGnGdgeh2LXEeSr9bJqruJxngsE7").get + ) + private[appender] def processAndBlacklistOnFailure[A, B]( ch: Channel, peerDatabase: PeerDatabase, @@ -96,13 +103,22 @@ package object appender extends ScorexLogging { _ <- validateTransactionSorting(height, block, settings.blockchainSettings.functionalitySettings) _ <- pos.validateBaseTarget(height, block, parent, grandParent) _ <- pos.validateGeneratorSignature(height, block) - _ <- pos.validateBlockDelay(height, block, parent, effectiveBalance) + _ <- pos.validateBlockDelay(height, block, parent, effectiveBalance).orElse(checkExceptions(height, block)) } yield () }.left.map { case GenericError(x) => GenericError(s"Block $block is invalid: $x") case x => x } + private def checkExceptions(height: Int, block: Block): Either[ValidationError, Unit] = { + Either + .cond( + exceptions.contains((height, block.uniqueId)), + (), + GenericError(s"Block time ${block.timestamp} less than expected") + ) + } + private def validateBlockVersion(height: Int, block: Block, fs: FunctionalitySettings): Either[ValidationError, Unit] = { val version3Height = fs.blockVersion3AfterHeight Either.cond( From b536ac599219ba665cf2d56a519236c33ebcbfaf Mon Sep 17 00:00:00 2001 From: peterz Date: Thu, 24 May 2018 16:57:12 +0300 Subject: [PATCH 44/52] NODE-772 Improve error reports in REST APIs that expect base64 --- lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala b/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala index 426201cfea7..78385a6f96e 100644 --- a/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala +++ b/lang/jvm/src/main/scala/com/wavesplatform/utils/Base64.scala @@ -6,7 +6,7 @@ object Base64 { def encode(input: Array[Byte]): String = "base64:" + new String(java.util.Base64.getEncoder.encode(input)) def decode(input: String): Try[Array[Byte]] = Try { - if (input.length < 7) throw new IllegalArgumentException("String of the form base64:chars expected") + if (!input.startsWith("base64:")) throw new IllegalArgumentException("String of the form base64:chars expected") else java.util.Base64.getDecoder.decode(input.substring(7)) } } From b4562bf1cf8c36fde03f1128962cb8bb8f43f08e Mon Sep 17 00:00:00 2001 From: Vyatcheslav Suharnikov Date: Mon, 21 May 2018 18:05:32 +0300 Subject: [PATCH 45/52] NODE-745 Offsets in expressions --- .../com/wavesplatform/lang/ParserTest.scala | 537 ++++++++++++------ .../lang/typechecker/CompilerV1Test.scala | 89 ++- .../lang/typechecker/ErrorTest.scala | 60 +- .../lang/v1/compiler/CompilerV1.scala | 58 +- .../lang/v1/parser/BinaryOperation.scala | 12 +- .../lang/v1/parser/Expressions.scala | 77 +-- .../wavesplatform/lang/v1/parser/Parser.scala | 233 ++++---- .../lang/v1/parser/UnaryOperation.scala | 31 +- .../lang/v1/testing/ScriptGen.scala | 110 ++-- .../smart/scenarios/OracleDataTest.scala | 18 +- 10 files changed, 717 insertions(+), 508 deletions(-) diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala index 8b77907ffd4..3285f7335cd 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala @@ -38,11 +38,6 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG throw new TestFailedException("Test failed", 0) } - private def isParsed(x: String): Boolean = Parser(x) match { - case Success(_, _) => true - case Failure(_, _, _) => false - } - private def genElementCheck(gen: Gen[EXPR]): Unit = { val testGen: Gen[(EXPR, String)] = for { expr <- gen @@ -81,46 +76,62 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG ) property("priority in binary expressions") { - parseOne("1 == 0 || 3 == 2") shouldBe BINARY_OP(BINARY_OP(CONST_LONG(1), EQ_OP, CONST_LONG(0)), - OR_OP, - BINARY_OP(CONST_LONG(3), EQ_OP, CONST_LONG(2))) - parseOne("3 + 2 > 2 + 1") shouldBe BINARY_OP(BINARY_OP(CONST_LONG(3), SUM_OP, CONST_LONG(2)), - GT_OP, - BINARY_OP(CONST_LONG(2), SUM_OP, CONST_LONG(1))) - parseOne("1 >= 0 || 3 > 2") shouldBe BINARY_OP(BINARY_OP(CONST_LONG(1), GE_OP, CONST_LONG(0)), - OR_OP, - BINARY_OP(CONST_LONG(3), GT_OP, CONST_LONG(2))) + parseOne("1 == 0 || 3 == 2") shouldBe BINARY_OP( + 0, + 16, + BINARY_OP(0, 6, CONST_LONG(0, 1, 1), EQ_OP, CONST_LONG(5, 6, 0)), + OR_OP, + BINARY_OP(10, 16, CONST_LONG(10, 11, 3), EQ_OP, CONST_LONG(15, 16, 2)) + ) + parseOne("3 + 2 > 2 + 1") shouldBe BINARY_OP( + 0, + 13, + BINARY_OP(0, 5, CONST_LONG(0, 1, 3), SUM_OP, CONST_LONG(4, 5, 2)), + GT_OP, + BINARY_OP(8, 13, CONST_LONG(8, 9, 2), SUM_OP, CONST_LONG(12, 13, 1)) + ) + parseOne("1 >= 0 || 3 > 2") shouldBe BINARY_OP( + 0, + 15, + BINARY_OP(0, 6, CONST_LONG(0, 1, 1), GE_OP, CONST_LONG(5, 6, 0)), + OR_OP, + BINARY_OP(10, 15, CONST_LONG(10, 11, 3), GT_OP, CONST_LONG(14, 15, 2)) + ) } property("bytestr expressions") { parseOne("false || sigVerify(base58'333', base58'222', base58'111')") shouldBe BINARY_OP( - FALSE, + 0, + 57, + FALSE(0, 5), OR_OP, FUNCTION_CALL( - "sigVerify", + 9, + 57, + PART.VALID(9, 18, "sigVerify"), List( - CONST_BYTEVECTOR(ByteVector(ScorexBase58.decode("333").get)), - CONST_BYTEVECTOR(ByteVector(ScorexBase58.decode("222").get)), - CONST_BYTEVECTOR(ByteVector(ScorexBase58.decode("111").get)) + CONST_BYTEVECTOR(19, 30, PART.VALID(27, 29, ByteVector(ScorexBase58.decode("333").get))), + CONST_BYTEVECTOR(32, 43, PART.VALID(40, 42, ByteVector(ScorexBase58.decode("222").get))), + CONST_BYTEVECTOR(45, 56, PART.VALID(53, 55, ByteVector(ScorexBase58.decode("111").get))) ) ) ) } property("valid non-empty base58 definition") { - parseOne("base58'bQbp'") shouldBe CONST_BYTEVECTOR(ByteVector("foo".getBytes)) + parseOne("base58'bQbp'") shouldBe CONST_BYTEVECTOR(0, 12, PART.VALID(8, 11, ByteVector("foo".getBytes))) } property("valid empty base58 definition") { - parseOne("base58''") shouldBe CONST_BYTEVECTOR(ByteVector.empty) + parseOne("base58''") shouldBe CONST_BYTEVECTOR(0, 8, PART.VALID(8, 7, ByteVector.empty)) } property("invalid base58 definition") { - parseOne("base58' bQbp'") shouldBe CONST_BYTEVECTOR(PART.INVALID(" bQbp", "Can't parse Base58 string")) + parseOne("base58' bQbp'") shouldBe CONST_BYTEVECTOR(0, 13, PART.INVALID(8, 12, "can't parse Base58 string")) } property("string is consumed fully") { - parseOne(""" " fooo bar" """) shouldBe CONST_STRING(" fooo bar") + parseOne(""" " fooo bar" """) shouldBe CONST_STRING(1, 17, PART.VALID(2, 16, " fooo bar")) } property("string literal with unicode chars") { @@ -132,47 +143,120 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG | "$stringWithUnicodeChars" | """.stripMargin - ) shouldBe CONST_STRING(stringWithUnicodeChars) + ) shouldBe CONST_STRING(3, 20, PART.VALID(4, 19, stringWithUnicodeChars)) } property("string literal with unicode chars in language") { - parseOne("\"\\u1234\"") shouldBe CONST_STRING("ሴ") + parseOne("\"\\u1234\"") shouldBe CONST_STRING(0, 8, PART.VALID(1, 7, "ሴ")) } property("should parse invalid unicode symbols") { - parseOne("\"\\uqwer\"") shouldBe CONST_STRING(PART.INVALID("\\uqwer", "Can't parse 'qwer' as HEX string in '\\uqwer'")) + parseOne("\"\\uqwer\"") shouldBe CONST_STRING( + 0, + 8, + PART.INVALID(1, 7, "can't parse 'qwer' as HEX string in '\\uqwer'") + ) } property("should parse incomplete unicode symbol definition") { - parseOne("\"\\u12 test\"") shouldBe CONST_STRING(PART.INVALID("\\u12 test", "Incomplete UTF-8 symbol definition: '\\u12'")) - parseOne("\"\\u\"") shouldBe CONST_STRING(PART.INVALID("\\u", "Incomplete UTF-8 symbol definition: '\\u'")) + parseOne("\"\\u12 test\"") shouldBe CONST_STRING(0, 11, PART.INVALID(1, 10, "incomplete UTF-8 symbol definition: '\\u12'")) + parseOne("\"\\u\"") shouldBe CONST_STRING(0, 4, PART.INVALID(1, 3, "incomplete UTF-8 symbol definition: '\\u'")) } property("string literal with special symbols") { - parseOne("\"\\t\"") shouldBe CONST_STRING("\t") + parseOne("\"\\t\"") shouldBe CONST_STRING(0, 4, PART.VALID(1, 3, "\t")) } property("should parse invalid special symbols") { - parseOne("\"\\ test\"") shouldBe CONST_STRING(PART.INVALID("\\ test", "Unknown escaped symbol: '\\ '")) + parseOne("\"\\ test\"") shouldBe CONST_STRING(0, 8, PART.INVALID(1, 7, "unknown escaped symbol: '\\ '. The valid are \b, \f, \n, \r, \t")) } property("should parse incomplete special symbols") { - parseOne("\"foo \\\"") shouldBe CONST_STRING(PART.INVALID("foo \\", "Invalid escaped symbol: '\\'")) + parseOne("\"foo \\\"") shouldBe CONST_STRING(0, 7, PART.INVALID(1, 6, "invalid escaped symbol: '\\'. The valid are \b, \f, \n, \r, \t")) + } + + property("reserved keywords are invalid variable names in block: if") { + val script = + s"""let if = 1 + |true""".stripMargin + parseOne(script) shouldBe BLOCK( + 0, + 15, + LET(0, 10, PART.INVALID(4, 6, "keywords are restricted"), CONST_LONG(9, 10, 1), Seq.empty), + TRUE(11, 15) + ) + } + + property("reserved keywords are invalid variable names in block: let") { + val script = + s"""let let = 1 + |true""".stripMargin + parseOne(script) shouldBe BLOCK( + 0, + 16, + LET(0, 11, PART.INVALID(4, 7, "keywords are restricted"), CONST_LONG(10, 11, 1), Seq.empty), + TRUE(12, 16) + ) } - property("reserved keywords are invalid variable names") { - List("if", "then", "else", "true", "false", "let").foreach { keyword => - val script = s"""let $keyword = 1 - |true""".stripMargin + List("then", "else", "true").foreach { keyword => + property(s"reserved keywords are invalid variable names in block: $keyword") { + val script = + s"""let ${keyword.padTo(4, " ").mkString} = 1 + |true""".stripMargin parseOne(script) shouldBe BLOCK( - LET(PART.INVALID(keyword, "keywords are restricted"), CONST_LONG(1), Seq.empty), - TRUE + 0, + 17, + LET(0, 12, PART.INVALID(4, 8, "keywords are restricted"), CONST_LONG(11, 12, 1), Seq.empty), + TRUE(13, 17) ) } + } + + property("reserved keywords are invalid variable names in block: false") { + val script = + s"""let false = 1 + |true""".stripMargin + parseOne(script) shouldBe BLOCK( + 0, + 18, + LET(0, 13, PART.INVALID(4, 9, "keywords are restricted"), CONST_LONG(12, 13, 1), Seq.empty), + TRUE(14, 18) + ) + } - List("if", "then", "else", "let").foreach { keyword => + property("reserved keywords are invalid variable names in expr: if") { + val script = "if + 1" + parseOne(script) shouldBe BINARY_OP( + 0, + 6, + REF(0, 2, PART.INVALID(0, 2, "keywords are restricted")), + BinaryOperation.SUM_OP, + CONST_LONG(5, 6, 1) + ) + } + + property("reserved keywords are invalid variable names in expr: let") { + val script = "let + 1" + parseOne(script) shouldBe BINARY_OP( + 0, + 7, + REF(0, 3, PART.INVALID(0, 3, "keywords are restricted")), + BinaryOperation.SUM_OP, + CONST_LONG(6, 7, 1) + ) + } + + List("then", "else").foreach { keyword => + property(s"reserved keywords are invalid variable names in expr: $keyword") { val script = s"$keyword + 1" - parseOne(script) shouldBe BINARY_OP(REF(PART.INVALID(keyword, "keywords are restricted")), BinaryOperation.SUM_OP, CONST_LONG(1)) + parseOne(script) shouldBe BINARY_OP( + 0, + 8, + REF(0, 4, PART.INVALID(0, 4, "keywords are restricted")), + BinaryOperation.SUM_OP, + CONST_LONG(7, 8, 1) + ) } } @@ -199,105 +283,130 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG } property("function call") { - parseOne("FOO(1,2)".stripMargin) shouldBe FUNCTION_CALL("FOO", List(CONST_LONG(1), CONST_LONG(2))) - parseOne("FOO(X)".stripMargin) shouldBe FUNCTION_CALL("FOO", List(REF("X"))) + parseOne("FOO(1,2)".stripMargin) shouldBe FUNCTION_CALL(0, 8, PART.VALID(0, 3, "FOO"), List(CONST_LONG(4, 5, 1), CONST_LONG(6, 7, 2))) + parseOne("FOO(X)".stripMargin) shouldBe FUNCTION_CALL(0, 6, PART.VALID(0, 3, "FOO"), List(REF(4, 5, PART.VALID(4, 5, "X")))) } property("function call on curly braces") { parseOne("{ 1 }(2, 3, 4)") shouldBe FUNCTION_CALL( - PART.INVALID("", "CONST_LONG(1) is not a function name"), - List(2, 3, 4).map(CONST_LONG(_)) + 0, + 14, + PART.INVALID(0, 5, "'CONST_LONG(2,3,1)' is not a function name"), + List(CONST_LONG(6, 7, 2), CONST_LONG(9, 10, 3), CONST_LONG(12, 13, 4)) ) } property("function call on round braces") { - parseOne("(1)(2, 3, 4)") shouldBe FUNCTION_CALL( - PART.INVALID("", "CONST_LONG(1) is not a function name"), - List(2, 3, 4).map(CONST_LONG(_)) + parseOne("( 1 )(2, 3, 4)") shouldBe FUNCTION_CALL( + 0, + 14, + PART.INVALID(0, 5, "'CONST_LONG(2,3,1)' is not a function name"), + List(CONST_LONG(6, 7, 2), CONST_LONG(9, 10, 3), CONST_LONG(12, 13, 4)) ) } - property("isDefined/extract") { - parseOne("isDefined(X)") shouldBe FUNCTION_CALL("isDefined", List(REF("X"))) + property("isDefined") { + parseOne("isDefined(X)") shouldBe FUNCTION_CALL(0, 12, PART.VALID(0, 9, "isDefined"), List(REF(10, 11, PART.VALID(10, 11, "X")))) + } + + property("extract") { parseOne("if(isDefined(X)) then extract(X) else Y") shouldBe IF( - FUNCTION_CALL("isDefined", List(REF("X"))), - FUNCTION_CALL("extract", List(REF("X"))), - REF("Y") + 0, + 39, + FUNCTION_CALL(3, 15, PART.VALID(3, 12, "isDefined"), List(REF(13, 14, PART.VALID(13, 14, "X")))), + FUNCTION_CALL(22, 32, PART.VALID(22, 29, "extract"), List(REF(30, 31, PART.VALID(30, 31, "X")))), + REF(38, 39, PART.VALID(38, 39, "Y")) ) } - property("getter") { - isParsed("xxx .yyy") shouldBe true - isParsed("xxx. yyy") shouldBe true + property("getter: spaces from left") { + parseOne("xxx .yyy") shouldBe GETTER(0, 9, REF(0, 3, PART.VALID(0, 3, "xxx")), PART.VALID(6, 9, "yyy")) + } - parseOne("xxx.yyy") shouldBe GETTER(REF("xxx"), "yyy") - parseOne( - """ - | - | xxx.yyy - | - """.stripMargin - ) shouldBe GETTER(REF("xxx"), "yyy") + property("getter: spaces from right") { + parseOne("xxx. yyy") shouldBe GETTER(0, 9, REF(0, 3, PART.VALID(0, 3, "xxx")), PART.VALID(6, 9, "yyy")) + } - parseOne("xxx(yyy).zzz") shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") - parseOne( - """ - | - | xxx(yyy).zzz - | - """.stripMargin - ) shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") + property("getter: no spaces") { + parseOne("xxx.yyy") shouldBe GETTER(0, 7, REF(0, 3, PART.VALID(0, 3, "xxx")), PART.VALID(4, 7, "yyy")) + } - parseOne("(xxx(yyy)).zzz") shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") - parseOne( - """ - | - | (xxx(yyy)).zzz - | - """.stripMargin - ) shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") + property("getter on function result") { + parseOne("xxx(yyy).zzz") shouldBe GETTER( + 0, + 12, + FUNCTION_CALL(0, 8, PART.VALID(0, 3, "xxx"), List(REF(4, 7, PART.VALID(4, 7, "yyy")))), + PART.VALID(9, 12, "zzz") + ) + } - parseOne("{xxx(yyy)}.zzz") shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") - parseOne( - """ - | - | { - | xxx(yyy) - | }.zzz - | - """.stripMargin - ) shouldBe GETTER(FUNCTION_CALL("xxx", List(REF("yyy"))), "zzz") + property("getter on round braces") { + parseOne("(xxx(yyy)).zzz") shouldBe GETTER( + 0, + 14, + FUNCTION_CALL(1, 9, PART.VALID(1, 4, "xxx"), List(REF(5, 8, PART.VALID(5, 8, "yyy")))), + PART.VALID(11, 14, "zzz") + ) + } + + property("getter on curly braces") { + parseOne("{xxx(yyy)}.zzz") shouldBe GETTER( + 0, + 14, + FUNCTION_CALL(1, 9, PART.VALID(1, 4, "xxx"), List(REF(5, 8, PART.VALID(5, 8, "yyy")))), + PART.VALID(11, 14, "zzz") + ) + } + property("getter on block") { parseOne( - """ - | - | { - | let yyy = aaa(bbb) - | xxx(yyy) - | }.zzz - | - """.stripMargin + """{ + | let yyy = aaa(bbb) + | xxx(yyy) + |}.zzz""".stripMargin ) shouldBe GETTER( + 0, + 39, BLOCK( - LET("yyy", FUNCTION_CALL("aaa", List(REF("bbb"))), Seq.empty), - FUNCTION_CALL("xxx", List(REF("yyy"))) + 4, + 33, + LET( + 4, + 22, + PART.VALID(8, 11, "yyy"), + FUNCTION_CALL(14, 22, PART.VALID(14, 17, "aaa"), List(REF(18, 21, PART.VALID(18, 21, "bbb")))), + Seq.empty + ), + FUNCTION_CALL(25, 33, PART.VALID(25, 28, "xxx"), List(REF(29, 32, PART.VALID(29, 32, "yyy")))) ), - "zzz" + PART.VALID(36, 39, "zzz") ) } - property("crypto functions") { - val hashFunctions = Vector("sha256", "blake2b256", "keccak256") - val text = "❤✓☀★☂♞☯☭☢€☎∞❄♫\u20BD=test message" - val encodedText = ScorexBase58.encode(text.getBytes) + // multiple getters - for (f <- hashFunctions) { - parseOne(s"$f(base58'$encodedText')".stripMargin) shouldBe - FUNCTION_CALL( - f, - List(CONST_BYTEVECTOR(ByteVector(text.getBytes))) - ) - } + property("crypto functions: sha256") { + val text = "❤✓☀★☂♞☯☭☢€☎∞❄♫\u20BD=test message" + val encodedText = ScorexBase58.encode(text.getBytes) + + parseOne(s"sha256(base58'$encodedText')".stripMargin) shouldBe + FUNCTION_CALL(0, 96, PART.VALID(0, 6, "sha256"), List(CONST_BYTEVECTOR(7, 95, PART.VALID(15, 94, ByteVector(text.getBytes))))) + } + + property("crypto functions: blake2b256") { + val text = "❤✓☀★☂♞☯☭☢€☎∞❄♫\u20BD=test message" + val encodedText = ScorexBase58.encode(text.getBytes) + + parseOne(s"blake2b256(base58'$encodedText')".stripMargin) shouldBe + FUNCTION_CALL(0, 100, PART.VALID(0, 10, "blake2b256"), List(CONST_BYTEVECTOR(11, 99, PART.VALID(19, 98, ByteVector(text.getBytes))))) + } + + property("crypto functions: keccak256") { + val text = "❤✓☀★☂♞☯☭☢€☎∞❄♫\u20BD=test message" + val encodedText = ScorexBase58.encode(text.getBytes) + + parseOne(s"keccak256(base58'$encodedText')".stripMargin) shouldBe + FUNCTION_CALL(0, 99, PART.VALID(0, 9, "keccak256"), List(CONST_BYTEVECTOR(10, 98, PART.VALID(18, 97, ByteVector(text.getBytes))))) } property("show parse all input including INVALID") { @@ -308,12 +417,9 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG |true""".stripMargin parseAll(script) shouldBe Seq( - BLOCK( - LET("C", CONST_LONG(1), Seq.empty), - REF("foo") - ), - INVALID("#@", CONST_LONG(2)), - TRUE + BLOCK(0, 13, LET(0, 9, PART.VALID(4, 5, "C"), CONST_LONG(8, 9, 1), Seq.empty), REF(10, 13, PART.VALID(10, 13, "foo"))), + INVALID(14, 16, "#@", Some(CONST_LONG(16, 17, 2))), + TRUE(18, 22) ) } @@ -323,8 +429,10 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG |# / |true""".stripMargin parseOne(script) shouldBe BLOCK( - LET("C", CONST_LONG(1), Seq.empty), - INVALID("#/", TRUE) + 0, + 18, + LET(0, 9, PART.VALID(4, 5, "C"), CONST_LONG(8, 9, 1), Seq.empty), + INVALID(10, 14, "# /\n", Some(TRUE(14, 18))) ) } @@ -334,10 +442,16 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG |let C = 1 |true""".stripMargin parseOne(script) shouldBe INVALID( - "#/", - BLOCK( - LET("C", CONST_LONG(1), Seq.empty), - TRUE + 0, + 4, + "# /\n", + Some( + BLOCK( + 4, + 18, + LET(4, 13, PART.VALID(8, 9, "C"), CONST_LONG(12, 13, 1), Seq.empty), + TRUE(14, 18) + ) ) ) } @@ -348,67 +462,86 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG |true |# /""".stripMargin parseAll(script) shouldBe Seq( - BLOCK( - LET("C", CONST_LONG(1), Seq.empty), - TRUE - ), - INVALID("#/") + BLOCK(0, 14, LET(0, 9, PART.VALID(4, 5, "C"), CONST_LONG(8, 9, 1), Seq.empty), TRUE(10, 14)), + INVALID(15, 18, "# /") ) } property("simple matching") { val code = - """ - | - | match tx { - | case a: TypeA => 0 - | case b: TypeB => 1 - | } - | - """.stripMargin - parseOne(code) shouldBe MATCH(REF("tx"), - List(MATCH_CASE(Some("a"), List("TypeA"), CONST_LONG(0)), MATCH_CASE(Some("b"), List("TypeB"), CONST_LONG(1)))) + """match tx { + | case a: TypeA => 0 + | case b: TypeB => 1 + |}""".stripMargin + parseOne(code) shouldBe MATCH( + 0, + 54, + REF(6, 8, PART.VALID(6, 8, "tx")), + List( + MATCH_CASE(13, 31, Some(PART.VALID(18, 19, "a")), List(PART.VALID(21, 26, "TypeA")), CONST_LONG(30, 31, 0)), + MATCH_CASE(34, 52, Some(PART.VALID(39, 40, "b")), List(PART.VALID(42, 47, "TypeB")), CONST_LONG(51, 52, 1)) + ) + ) } property("multiple union type matching") { val code = - """ - | - | match tx { - | case txa: TypeA => 0 - | case underscore : TypeB | TypeC => 1 - | } - | - """.stripMargin - parseOne(code) shouldBe MATCH(REF("tx"), - List(MATCH_CASE(Some("txa"), List("TypeA"), CONST_LONG(0)), - MATCH_CASE(Some("underscore"), List("TypeB", "TypeC"), CONST_LONG(1)))) + """match tx { + | case txa: TypeA => 0 + | case underscore : TypeB | TypeC => 1 + |}""".stripMargin + parseOne(code) shouldBe MATCH( + 0, + 74, + REF(6, 8, PART.VALID(6, 8, "tx")), + List( + MATCH_CASE(13, 33, Some(PART.VALID(18, 21, "txa")), List(PART.VALID(23, 28, "TypeA")), CONST_LONG(32, 33, 0)), + MATCH_CASE( + 36, + 72, + Some(PART.VALID(41, 51, "underscore")), + List(PART.VALID(54, 59, "TypeB"), PART.VALID(62, 67, "TypeC")), + CONST_LONG(71, 72, 1) + ) + ) + ) } property("matching expression") { val code = - """ - | - | match foo(x) + bar { - | case x:TypeA => 0 - | case y:TypeB | TypeC => 1 - | } - | - """.stripMargin + """match foo(x) + bar { + | case x:TypeA => 0 + | case y:TypeB | TypeC => 1 + |}""".stripMargin parseOne(code) shouldBe MATCH( - BINARY_OP(FUNCTION_CALL("foo", List(REF("x"))), BinaryOperation.SUM_OP, REF("bar")), - List(MATCH_CASE(Some("x"), List("TypeA"), CONST_LONG(0)), MATCH_CASE(Some("y"), List("TypeB", "TypeC"), CONST_LONG(1))) + 0, + 70, + BINARY_OP( + 6, + 18, + FUNCTION_CALL(6, 12, PART.VALID(6, 9, "foo"), List(REF(10, 11, PART.VALID(10, 11, "x")))), + BinaryOperation.SUM_OP, + REF(15, 18, PART.VALID(15, 18, "bar")) + ), + List( + MATCH_CASE(23, 40, Some(PART.VALID(28, 29, "x")), List(PART.VALID(30, 35, "TypeA")), CONST_LONG(39, 40, 0)), + MATCH_CASE(43, 68, Some(PART.VALID(48, 49, "y")), List(PART.VALID(50, 55, "TypeB"), PART.VALID(58, 63, "TypeC")), CONST_LONG(67, 68, 1)) + ) ) } property("pattern matching with valid case, but no type is defined") { parseOne("match tx { case x => 1 } ") shouldBe MATCH( - REF("tx"), + 0, + 24, + REF(6, 8, PART.VALID(6, 8, "tx")), List( MATCH_CASE( - Some(PART.VALID("x")), + 11, + 22, + Some(PART.VALID(16, 17, "x")), List.empty, - CONST_LONG(1) + CONST_LONG(21, 22, 1) ) ) ) @@ -416,29 +549,37 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG property("pattern matching with valid case, placeholder instead of variable name") { parseOne("match tx { case _:TypeA => 1 } ") shouldBe MATCH( - REF("tx"), + 0, + 31, + REF(6, 8, PART.VALID(6, 8, "tx")), List( MATCH_CASE( + 11, + 29, None, - List(PART.VALID("TypeA")), - CONST_LONG(1) + List(PART.VALID(19, 24, "TypeA")), + CONST_LONG(28, 29, 1) ) ) ) } property("pattern matching with no cases") { - parseOne("match tx { } ") shouldBe INVALID("pattern matching requires case branches") + parseOne("match tx { } ") shouldBe INVALID(0, 12, "pattern matching requires case branches") } property("pattern matching with invalid case - no variable, type and expr are defined") { parseOne("match tx { case => } ") shouldBe MATCH( - REF("tx"), + 0, + 20, + REF(6, 8, PART.VALID(6, 8, "tx")), List( MATCH_CASE( - Some(PART.INVALID("", "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), + 11, + 18, + Some(PART.INVALID(16, 16, "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), List.empty, - INVALID("expected expression") + INVALID(16, 18, "expected expression") ) ) ) @@ -446,12 +587,16 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG property("pattern matching with invalid case - no variable and type are defined") { parseOne("match tx { case => 1 } ") shouldBe MATCH( - REF("tx"), + 0, + 22, + REF(6, 8, PART.VALID(6, 8, "tx")), List( MATCH_CASE( - Some(PART.INVALID("", "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), + 11, + 20, + Some(PART.INVALID(16, 16, "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), List.empty, - CONST_LONG(1) + CONST_LONG(19, 20, 1) ) ) ) @@ -459,25 +604,27 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG property("pattern matching with invalid case - no expr is defined") { parseOne("match tx { case TypeA => } ") shouldBe MATCH( - REF("tx"), + 0, + 26, + REF(6, 8, PART.VALID(6, 8, "tx")), List( - MATCH_CASE( - Some(PART.VALID("TypeA")), - Seq.empty, - INVALID("expected expression") - ) + MATCH_CASE(11, 24, Some(PART.VALID(16, 21, "TypeA")), Seq.empty, INVALID(21, 24, "expected expression")) ) ) } property("pattern matching with invalid case - no var is defined") { parseOne("match tx { case :TypeA => 1 } ") shouldBe MATCH( - REF("tx"), + 0, + 29, + REF(6, 8, PART.VALID(6, 8, "tx")), List( MATCH_CASE( - Some(PART.INVALID(":TypeA ", "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), + 11, + 27, + Some(PART.INVALID(16, 23, "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), Seq.empty, - CONST_LONG(1) + CONST_LONG(26, 27, 1) ) ) ) @@ -485,12 +632,16 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG property("pattern matching with invalid case - expression in variable definition") { parseOne("match tx { case 1 + 1 => 1 } ") shouldBe MATCH( - REF("tx"), + 0, + 28, + REF(6, 8, PART.VALID(6, 8, "tx")), List( MATCH_CASE( - Some(PART.INVALID("1 + 1 ", "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), + 11, + 26, + Some(PART.INVALID(16, 22, "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), List.empty, - CONST_LONG(1) + CONST_LONG(25, 26, 1) ) ) ) @@ -498,12 +649,16 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG property("pattern matching with default case - no type is defined, one separator") { parseOne("match tx { case _: | => 1 } ") shouldBe MATCH( - REF("tx"), + 0, + 27, + REF(6, 8, PART.VALID(6, 8, "tx")), List( MATCH_CASE( + 11, + 25, None, - Seq(PART.INVALID("| ", "the type for variable should be specified: `case varName: Type => expr`")), - CONST_LONG(1) + Seq(PART.INVALID(19, 21, "the type for variable should be specified: `case varName: Type => expr`")), + CONST_LONG(24, 25, 1) ) ) ) @@ -511,12 +666,16 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG property("pattern matching with default case - no type is defined, multiple separators") { parseOne("match tx { case _: |||| => 1 } ") shouldBe MATCH( - REF("tx"), + 0, + 31, + REF(6, 8, PART.VALID(6, 8, "tx")), List( MATCH_CASE( + 11, + 29, None, - Seq(PART.INVALID("|||| ", "the type for variable should be specified: `case varName: Type => expr`")), - CONST_LONG(1) + Seq(PART.INVALID(20, 25, "the type for variable should be specified: `case varName: Type => expr`")), + CONST_LONG(28, 29, 1) ) ) ) diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala index ffec93c6878..a47385557fd 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala @@ -16,28 +16,35 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr property("should infer generic function return type") { import com.wavesplatform.lang.v1.parser.Expressions._ - val Right(v) = CompilerV1(typeCheckerContext, FUNCTION_CALL(idT.name, List(CONST_LONG(1)))) + val Right(v) = CompilerV1(typeCheckerContext, FUNCTION_CALL(0, 0, PART.VALID(0, 0, idT.name), List(CONST_LONG(0, 0, 1)))) v.tpe shouldBe LONG } property("should infer inner types") { import com.wavesplatform.lang.v1.parser.Expressions._ val Right(v) = - CompilerV1(typeCheckerContext, FUNCTION_CALL(extract.name, List(FUNCTION_CALL(undefinedOptionLong.name, List.empty)))) + CompilerV1( + typeCheckerContext, + FUNCTION_CALL(0, 0, PART.VALID(0, 0, extract.name), List(FUNCTION_CALL(0, 0, PART.VALID(0, 0, undefinedOptionLong.name), List.empty))) + ) v.tpe shouldBe LONG } treeTypeTest(s"unitOnNone(NONE)")( ctx = typeCheckerContext, - expr = Expressions.FUNCTION_CALL(unitOnNone.name, List(Expressions.REF("None"))), + expr = Expressions.FUNCTION_CALL(0, + 0, + Expressions.PART.VALID(0, 0, unitOnNone.name), + List(Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "None")))), expectedResult = Right(FUNCTION_CALL(unitOnNone.header, List(REF("None", OPTION(NOTHING))), UNIT)) ) property("successful on very deep expressions(stack overflow check)") { - val expr = - (1 to 100000).foldLeft[Expressions.EXPR](Expressions.CONST_LONG(0))((acc, _) => Expressions.BINARY_OP(acc, SUM_OP, Expressions.CONST_LONG(1))) - val expectedResult = Right(LONG) + val expr = (1 to 100000).foldLeft[Expressions.EXPR](Expressions.CONST_LONG(0, 0, 0)) { (acc, _) => + Expressions.BINARY_OP(0, 0, acc, SUM_OP, Expressions.CONST_LONG(0, 0, 1)) + } + val expectedResult = Right(LONG) CompilerV1(typeCheckerContext, expr).map(_.tpe) match { case Right(x) => Right(x) shouldBe expectedResult case e @ Left(_) => e shouldBe expectedResult @@ -47,8 +54,10 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr treeTypeTest("GETTER")( ctx = CompilerContext(predefTypes = Map(pointType.name -> pointType), varDefs = Map("p" -> TYPEREF("Point")), functionDefs = Map.empty), expr = Expressions.GETTER( - ref = Expressions.REF("p"), - field = "x" + 0, + 0, + ref = Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "p")), + field = Expressions.PART.VALID(0, 0, "x") ), expectedResult = Right( GETTER( @@ -60,31 +69,53 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr treeTypeTest("REF(OBJECT)")( ctx = CompilerContext(predefTypes = Map(pointType.name -> pointType), varDefs = Map("p" -> TYPEREF("Point")), functionDefs = Map.empty), - expr = Expressions.REF("p"), + expr = Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "p")), expectedResult = Right(REF("p", TYPEREF("Point"))) ) treeTypeTest("REF x = y")( ctx = CompilerContext(predefTypes = Map(pointType.name -> pointType), varDefs = Map("p" -> TYPEREF("Point")), functionDefs = Map.empty), - expr = Expressions.REF("p"), + expr = Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "p")), expectedResult = Right(REF("p", TYPEREF("Point"))) ) treeTypeTest("MULTIPLY(1,2)")( ctx = typeCheckerContext, - expr = Expressions.FUNCTION_CALL(multiplierFunction.name, List(Expressions.CONST_LONG(1), Expressions.CONST_LONG(2))), + expr = Expressions.FUNCTION_CALL( + 0, + 0, + Expressions.PART.VALID(0, 0, multiplierFunction.name), + List(Expressions.CONST_LONG(0, 0, 1), Expressions.CONST_LONG(0, 0, 2)) + ), expectedResult = Right(FUNCTION_CALL(multiplierFunction.header, List(CONST_LONG(1), CONST_LONG(2)), LONG)) ) treeTypeTest(s"idOptionLong(NONE)")( ctx = typeCheckerContext, - expr = Expressions.FUNCTION_CALL(idOptionLong.name, List(Expressions.REF("None"))), + expr = Expressions.FUNCTION_CALL( + 0, + 0, + Expressions.PART.VALID(0, 0, idOptionLong.name), + List(Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "None"))) + ), expectedResult = Right(FUNCTION_CALL(idOptionLong.header, List(REF("None", OPTION(NOTHING))), UNIT)) ) treeTypeTest(s"idOptionLong(SOME(NONE))")( ctx = typeCheckerContext, - expr = Expressions.FUNCTION_CALL(idOptionLong.name, List(Expressions.FUNCTION_CALL("Some", List(Expressions.REF("None"))))), + expr = Expressions.FUNCTION_CALL( + 0, + 0, + Expressions.PART.VALID(0, 0, idOptionLong.name), + List( + Expressions.FUNCTION_CALL( + 0, + 0, + Expressions.PART.VALID(0, 0, "Some"), + List(Expressions.REF(0, 0, Expressions.PART.INVALID(0, 0, "None"))) + ) + ) + ), expectedResult = Right(FUNCTION_CALL(idOptionLong.header, List(FUNCTION_CALL(some.header, List(REF("None", OPTION(NOTHING))), OPTION(OPTION(NOTHING)))), UNIT)) ) @@ -92,8 +123,17 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr treeTypeTest(s"idOptionLong(SOME(CONST_LONG(3)))")( ctx = typeCheckerContext, expr = Expressions.FUNCTION_CALL( - idOptionLong.name, - List(Expressions.FUNCTION_CALL("Some", List(Expressions.FUNCTION_CALL("Some", List(Expressions.CONST_LONG(3)))))) + 0, + 0, + Expressions.PART.VALID(0, 0, idOptionLong.name), + List( + Expressions.FUNCTION_CALL( + 0, + 0, + Expressions.PART.VALID(0, 0, "Some"), + List(Expressions.FUNCTION_CALL(0, 0, Expressions.PART.VALID(0, 0, "Some"), List(Expressions.CONST_LONG(0, 0, 3)))) + ) + ) ), expectedResult = Right( FUNCTION_CALL( @@ -106,43 +146,48 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr treeTypeTest("Invalid LET")( ctx = typeCheckerContext, - expr = Expressions.BLOCK(Expressions.LET(Expressions.PART.INVALID("###", "it is invalid!"), Expressions.TRUE, Seq.empty), Expressions.REF("x")), + expr = Expressions.BLOCK( + 0, + 0, + Expressions.LET(0, 0, Expressions.PART.INVALID(0, 0, "it is invalid!"), Expressions.TRUE(0, 0), Seq.empty), + Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "x")) + ), expectedResult = Left("Typecheck failed: it is invalid!: ###") ) treeTypeTest("Invalid GETTER")( ctx = typeCheckerContext, - expr = Expressions.GETTER(Expressions.REF("x"), Expressions.PART.INVALID("###", "it is invalid!")), + expr = Expressions.GETTER(0, 0, Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "x")), Expressions.PART.INVALID(0, 0, "it is invalid!")), expectedResult = Left("Typecheck failed: it is invalid!: ###") ) treeTypeTest("Invalid BYTEVECTOR")( ctx = typeCheckerContext, - expr = Expressions.CONST_BYTEVECTOR(Expressions.PART.INVALID("foo", "it is invalid!")), + expr = Expressions.CONST_BYTEVECTOR(0, 0, Expressions.PART.INVALID(0, 0, "it is invalid!")), expectedResult = Left("Typecheck failed: it is invalid!: foo") ) treeTypeTest("Invalid STRING")( ctx = typeCheckerContext, - expr = Expressions.CONST_STRING(Expressions.PART.INVALID("\\u1", "it is invalid!")), + expr = Expressions.CONST_STRING(0, 0, Expressions.PART.INVALID(0, 0, "it is invalid!")), expectedResult = Left("Typecheck failed: it is invalid!: \\u1") ) treeTypeTest("Invalid REF")( ctx = typeCheckerContext, - expr = Expressions.REF(Expressions.PART.INVALID("###", "it is invalid!")), + expr = Expressions.REF(0, 0, Expressions.PART.INVALID(0, 0, "it is invalid!")), expectedResult = Left("Typecheck failed: it is invalid!: ###") ) treeTypeTest("Invalid FUNCTION_CALL")( ctx = typeCheckerContext, - expr = Expressions.FUNCTION_CALL(Expressions.PART.INVALID("###", "it is invalid!"), List.empty), + expr = Expressions.FUNCTION_CALL(0, 0, Expressions.PART.INVALID(0, 0, "it is invalid!"), List.empty), expectedResult = Left("Typecheck failed: it is invalid!: ###") ) treeTypeTest("INVALID")( ctx = typeCheckerContext, - expr = Expressions.INVALID("###", None), + expr = Expressions.INVALID(0, 0, "###", None), expectedResult = Left("Typecheck failed: ###") ) diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/ErrorTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/ErrorTest.scala index e16f473aec6..efd96106e1f 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/ErrorTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/ErrorTest.scala @@ -15,24 +15,50 @@ class ErrorTest extends PropSpec with PropertyChecks with Matchers with ScriptGe errorTests( "can't define LET with the same name as already defined in scope" -> "already defined in the scope" -> BLOCK( - LET("X", CONST_LONG(1), Seq.empty), - BLOCK(LET("X", CONST_LONG(2), Seq.empty), TRUE)), - "can't define LET with the same name as predefined constant" -> "already defined in the scope" -> BLOCK(LET("None", CONST_LONG(2), Seq.empty), - TRUE), + 0, + 0, + LET(0, 0, PART.VALID(0, 0, "X"), CONST_LONG(0, 0, 1), Seq.empty), + BLOCK(0, 0, LET(0, 0, PART.VALID(0, 0, "X"), CONST_LONG(0, 0, 2), Seq.empty), TRUE(0, 0)) + ), + "can't define LET with the same name as predefined constant" -> "already defined in the scope" -> BLOCK( + 0, + 0, + LET(0, 0, PART.VALID(0, 0, "None"), CONST_LONG(0, 0, 2), Seq.empty), + TRUE(0, 0) + ), "can't define LET with the same name as predefined function" -> "function with such name is predefined" -> BLOCK( - LET("Some", CONST_LONG(2), Seq.empty), - TRUE), - "BINARY_OP with wrong types" -> "Typecheck failed: Can't find a function '+'" -> BINARY_OP(TRUE, SUM_OP, CONST_LONG(1)), - "IF can't find common" -> "Can't find common type" -> IF(TRUE, TRUE, CONST_LONG(0)), - "IF clause must be boolean" -> "IF clause is expected to be BOOLEAN" -> IF(CONST_LONG(0), TRUE, FALSE), - "FUNCTION_CALL with wrong amount of arguments" -> "requires 2 arguments" -> FUNCTION_CALL(multiplierFunction.name, List(CONST_LONG(0))), - "FUNCTION_CALL with upper type" -> "Non-matching types" -> FUNCTION_CALL(unitOnNone.name, List(FUNCTION_CALL("Some", List(CONST_LONG(3))))), - "FUNCTION_CALL with wrong type of argument" -> "Typecheck failed: Non-matching types: expected: LONG, actual: BOOLEAN" -> FUNCTION_CALL( - multiplierFunction.name, - List(CONST_LONG(0), FALSE)), - "FUNCTION_CALL with uncommon types for parameter T" -> "Can't match inferred types" -> FUNCTION_CALL(functionWithTwoPrarmsOfTheSameType.name, - List(CONST_LONG(1), - CONST_BYTEVECTOR(ByteVector.empty))) + 0, + 0, + LET(0, 0, PART.VALID(0, 0, "Some"), CONST_LONG(0, 0, 2), Seq.empty), + TRUE(0, 0) + ), + "BINARY_OP with wrong types" -> "Typecheck failed: Can't find a function '+'" -> BINARY_OP(0, 0, TRUE(0, 0), SUM_OP, CONST_LONG(0, 0, 1)), + "IF can't find common" -> "Can't find common type" -> IF(0, 0, TRUE(0, 0), TRUE(0, 0), CONST_LONG(0, 0, 0)), + "IF clause must be boolean" -> "IF clause is expected to be BOOLEAN" -> IF(0, 0, CONST_LONG(0, 0, 0), TRUE(0, 0), FALSE(0, 0)), + "FUNCTION_CALL with wrong amount of arguments" -> "requires 2 arguments" -> FUNCTION_CALL( + 0, + 0, + PART.VALID(0, 0, multiplierFunction.name), + List(CONST_LONG(0, 0, 0)) + ), + "FUNCTION_CALL with upper type" -> "Non-matching types" -> FUNCTION_CALL( + 0, + 0, + PART.VALID(0, 0, unitOnNone.name), + List(FUNCTION_CALL(0, 0, PART.VALID(0, 0, "Some"), List(CONST_LONG(0, 0, 3)))) + ), + "FUNCTION_CALL with wrong type of argument" -> "Typecheck failed: Non-matching types: expected: LONG, actual: BOOLEAN" -> FUNCTION_CALL( + 0, + 0, + PART.VALID(0, 0, multiplierFunction.name), + List(CONST_LONG(0, 0, 0), FALSE(0, 0)) + ), + "FUNCTION_CALL with uncommon types for parameter T" -> "Can't match inferred types" -> FUNCTION_CALL( + 0, + 0, + PART.VALID(0, 0, functionWithTwoPrarmsOfTheSameType.name), + List(CONST_LONG(0, 0, 1), CONST_BYTEVECTOR(0, 0, PART.VALID(0, 0, ByteVector.empty))) + ) ) private def errorTests(exprs: ((String, String), Expressions.EXPR)*): Unit = exprs.foreach { diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala index c7b51e9e7a9..152d5f42282 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala @@ -48,23 +48,23 @@ object CompilerV1 { private def compile(ctx: CompilerContext, t: SetTypeResult[Expressions.EXPR]): SetTypeResult[EXPR] = t.flatMap { case x: Expressions.CONST_LONG => EitherT.pure(CONST_LONG(x.value)) - case Expressions.CONST_BYTEVECTOR(p) => handlePart(p)(CONST_BYTEVECTOR) - case Expressions.CONST_STRING(p) => handlePart(p)(CONST_STRING) - case Expressions.TRUE => EitherT.pure(TRUE) - case Expressions.FALSE => EitherT.pure(FALSE) + case x: Expressions.CONST_BYTEVECTOR => handlePart(x.value)(CONST_BYTEVECTOR) + case x: Expressions.CONST_STRING => handlePart(x.value)(CONST_STRING) + case _: Expressions.TRUE => EitherT.pure(TRUE) + case _: Expressions.FALSE => EitherT.pure(FALSE) case getter: Expressions.GETTER => compileGetter(ctx, getter) case fc: Expressions.FUNCTION_CALL => compileFunctionCall(ctx, fc) case block: Expressions.BLOCK => compileBlock(ctx, block) case ifExpr: Expressions.IF => compileIf(ctx, ifExpr) case ref: Expressions.REF => compileRef(ctx, ref) case m: Expressions.MATCH => compileMatch(ctx, m) - case Expressions.BINARY_OP(a, op, b) => + case Expressions.BINARY_OP(start, end, a, op, b) => op match { - case AND_OP => compileIf(ctx, Expressions.IF(a, b, Expressions.FALSE)) - case OR_OP => compileIf(ctx, Expressions.IF(a, Expressions.TRUE, b)) - case _ => compileFunctionCall(ctx, Expressions.FUNCTION_CALL(opsToFunctions(op), List(a, b))) + case AND_OP => compileIf(ctx, Expressions.IF(start, end, a, b, Expressions.FALSE(start, end))) + case OR_OP => compileIf(ctx, Expressions.IF(start, end, a, Expressions.TRUE(start, end), b)) + case _ => compileFunctionCall(ctx, Expressions.FUNCTION_CALL(start, end, PART.VALID(start, end, opsToFunctions(op)), List(a, b))) } - case Expressions.INVALID(message, _) => EitherT.leftT[Coeval, EXPR](message) + case Expressions.INVALID(_, _, message, _) => EitherT.leftT[Coeval, EXPR](message) } private def compileGetter(ctx: CompilerContext, getter: Expressions.GETTER): SetTypeResult[EXPR] = @@ -124,7 +124,7 @@ object CompilerV1 { } private def compileFunctionCall(ctx: CompilerContext, fc: Expressions.FUNCTION_CALL): SetTypeResult[EXPR] = { - val Expressions.FUNCTION_CALL(name, args) = fc + val Expressions.FUNCTION_CALL(_, _, name, args) = fc for { name <- EitherT.fromEither[Coeval](name.toEither) r <- ctx.functionTypeSignaturesByName(name) match { @@ -185,9 +185,9 @@ object CompilerV1 { } private def compileMatch(ctx: CompilerContext, m: Expressions.MATCH): SetTypeResult[EXPR] = { - val Expressions.MATCH(expr, cases) = m - val rootMatchTmpArg = "$match" + ctx.tmpArgsIdx - val updatedCtx = ctx.copy(tmpArgsIdx = ctx.tmpArgsIdx + 1) + val Expressions.MATCH(_, _, expr, cases) = m + val rootMatchTmpArg = "$match" + ctx.tmpArgsIdx + val updatedCtx = ctx.copy(tmpArgsIdx = ctx.tmpArgsIdx + 1) for { typedExpr <- compile(ctx, EitherT.pure(expr)) @@ -203,24 +203,32 @@ object CompilerV1 { (), s"Matching not exhaustive: possibleTypes are ${possibleExpressionTypes.l}, while matched are $matchingTypes" ) - refTmp = Expressions.REF(rootMatchTmpArg) - ifBasedCases: Expressions.EXPR = cases.foldRight(Expressions.REF(PureContext.errRef): Expressions.EXPR) { + refTmp = Expressions.REF(1, 1, PART.VALID(1, 1, rootMatchTmpArg)) + ifBasedCases: Expressions.EXPR = cases.foldRight(Expressions.REF(1, 1, PART.VALID(1, 1, PureContext.errRef)): Expressions.EXPR) { case (mc, further) => - val typeSwarma = mc.types.foldLeft(Expressions.FALSE: Expressions.EXPR) { + val typeSwarma = mc.types.foldLeft(Expressions.FALSE(1, 1): Expressions.EXPR) { case (other, matchType) => - BINARY_OP(Expressions.FUNCTION_CALL(PureContext._isInstanceOf.name, List(refTmp, Expressions.CONST_STRING(matchType))), - BinaryOperation.OR_OP, - other) + BINARY_OP( + 1, + 1, + Expressions.FUNCTION_CALL(1, + 1, + PART.VALID(1, 1, PureContext._isInstanceOf.name), + List(refTmp, Expressions.CONST_STRING(1, 1, matchType))), + BinaryOperation.OR_OP, + other + ) } val blockWithNewVar = mc.newVarName match { - case Some(newVal) => Expressions.BLOCK(Expressions.LET(newVal, refTmp, mc.types), mc.expr) + case Some(newVal) => Expressions.BLOCK(1, 1, Expressions.LET(1, 1, newVal, refTmp, mc.types), mc.expr) case None => mc.expr } - if (typeSwarma == Expressions.FALSE) + if (typeSwarma.isInstanceOf[Expressions.FALSE]) blockWithNewVar - else Expressions.IF(typeSwarma, blockWithNewVar, further) + else Expressions.IF(1, 1, typeSwarma, blockWithNewVar, further) } - compiled <- compileBlock(updatedCtx, Expressions.BLOCK(Expressions.LET(rootMatchTmpArg, expr, Seq.empty), ifBasedCases)) + compiled <- compileBlock(updatedCtx, + Expressions.BLOCK(1, 1, Expressions.LET(1, 1, PART.VALID(1, 1, rootMatchTmpArg), expr, Seq.empty), ifBasedCases)) } yield compiled } @@ -255,8 +263,8 @@ object CompilerV1 { } private def handlePart[T](part: PART[T])(f: T => EXPR): SetTypeResult[EXPR] = part match { - case PART.VALID(x) => EitherT.pure(f(x)) - case PART.INVALID(x, message) => EitherT.leftT[Coeval, EXPR](s"$message: $x") + case PART.VALID(_, _, x) => EitherT.pure(f(x)) + case PART.INVALID(start, end, message) => EitherT.leftT[Coeval, EXPR](s"$message at $start-$end") } def apply(c: CompilerContext, expr: Expressions.EXPR): CompilationResult[EXPR] = { diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala index ea8d5079d49..32c4113c086 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/BinaryOperation.scala @@ -6,8 +6,8 @@ import fastparse.all._ sealed abstract class BinaryOperation { val func: String val parser: P[Any] = P(func) - def expr(op1: EXPR)(op2: EXPR): EXPR = { - BINARY_OP(op1, this, op2) + def expr(start: Int, end: Int, op1: EXPR, op2: EXPR): EXPR = { + BINARY_OP(start, end, op1, this, op2) } } @@ -55,15 +55,15 @@ object BinaryOperation { case object LE_OP extends BinaryOperation { val func = ">=" override val parser = P("<=") - override def expr(op1: EXPR)(op2: EXPR): EXPR = { - BINARY_OP(op2, LE_OP, op1) + override def expr(start: Int, end: Int, op1: EXPR, op2: EXPR): EXPR = { + BINARY_OP(start, end, op2, LE_OP, op1) } } case object LT_OP extends BinaryOperation { val func = ">" override val parser = P("<") - override def expr(op1: EXPR)(op2: EXPR): EXPR = { - BINARY_OP(op2, LT_OP, op1) + override def expr(start: Int, end: Int, op1: EXPR, op2: EXPR): EXPR = { + BINARY_OP(start, end, op2, LT_OP, op1) } } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala index 75bd7ff1dbe..9e02e03d853 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala @@ -4,69 +4,42 @@ import scodec.bits.ByteVector object Expressions { - sealed trait PART[+T] - object PART { - case class VALID[T](v: T) extends PART[T] - case class INVALID(consumed: String, message: String) extends PART[Nothing] - } - - case class LET(name: PART[String], value: EXPR, types: Seq[PART[String]]) - object LET { - def apply(name: String, value: EXPR, types: Seq[String]): LET = LET(PART.VALID(name), value, types.map(PART.VALID(_))) - } - - sealed trait EXPR - case class CONST_LONG(value: Long) extends EXPR - - case class GETTER(ref: EXPR, field: PART[String]) extends EXPR - object GETTER { - def apply(ref: EXPR, field: String): GETTER = GETTER(ref, PART.VALID(field)) - } - - case class CONST_BYTEVECTOR(value: PART[ByteVector]) extends EXPR - object CONST_BYTEVECTOR { - def apply(x: ByteVector): CONST_BYTEVECTOR = CONST_BYTEVECTOR(PART.VALID(x)) + trait Positioned { + def start: Int + def end: Int } - case class CONST_STRING(value: PART[String]) extends EXPR - object CONST_STRING { - def apply(x: String): CONST_STRING = CONST_STRING(PART.VALID(x)) + sealed trait PART[+T] extends Positioned + object PART { + case class VALID[T](start: Int, end: Int, v: T) extends PART[T] + case class INVALID(start: Int, end: Int, message: String) extends PART[Nothing] } - case class BINARY_OP(a: EXPR, kind: BinaryOperation, b: EXPR) extends EXPR - case class BLOCK(let: LET, body: EXPR) extends EXPR - case class IF(cond: EXPR, ifTrue: EXPR, ifFalse: EXPR) extends EXPR + case class LET(start: Int, end: Int, name: PART[String], value: EXPR, types: Seq[PART[String]]) extends Positioned - case class REF(key: PART[String]) extends EXPR - object REF { - def apply(key: String): REF = REF(PART.VALID(key)) - } + sealed trait EXPR extends Positioned + case class CONST_LONG(start: Int, end: Int, value: Long) extends EXPR + case class GETTER(start: Int, end: Int, ref: EXPR, field: PART[String]) extends EXPR + case class CONST_BYTEVECTOR(start: Int, end: Int, value: PART[ByteVector]) extends EXPR + case class CONST_STRING(start: Int, end: Int, value: PART[String]) extends EXPR + case class BINARY_OP(start: Int, end: Int, a: EXPR, kind: BinaryOperation, b: EXPR) extends EXPR + case class BLOCK(start: Int, end: Int, let: LET, body: EXPR) extends EXPR + case class IF(start: Int, end: Int, cond: EXPR, ifTrue: EXPR, ifFalse: EXPR) extends EXPR + case class REF(start: Int, end: Int, key: PART[String]) extends EXPR - case object TRUE extends EXPR - case object FALSE extends EXPR + case class TRUE(start: Int, end: Int) extends EXPR + case class FALSE(start: Int, end: Int) extends EXPR - case class FUNCTION_CALL(name: PART[String], args: List[EXPR]) extends EXPR - object FUNCTION_CALL { - def apply(name: String, args: List[EXPR]): FUNCTION_CALL = FUNCTION_CALL(PART.VALID(name), args) - } + case class FUNCTION_CALL(start: Int, end: Int, name: PART[String], args: List[EXPR]) extends EXPR - case class MATCH_CASE(newVarName: Option[PART[String]], types: Seq[PART[String]], expr: EXPR) - object MATCH_CASE { - def apply(newVarName: Option[String], types: List[String], expr: EXPR): MATCH_CASE = - MATCH_CASE(newVarName.map(PART.VALID(_)), types.map(PART.VALID(_)), expr) - } - - case class MATCH(expr: EXPR, cases: Seq[MATCH_CASE]) extends EXPR - - case class INVALID(message: String, next: Option[EXPR] = None) extends EXPR - object INVALID { - def apply(message: String, next: EXPR): INVALID = INVALID(message, Some(next)) - } + case class MATCH_CASE(start: Int, end: Int, newVarName: Option[PART[String]], types: Seq[PART[String]], expr: EXPR) + case class MATCH(start: Int, end: Int, expr: EXPR, cases: Seq[MATCH_CASE]) extends EXPR + case class INVALID(start: Int, end: Int, message: String, next: Option[EXPR] = None) extends EXPR implicit class PartOps[T](val self: PART[T]) extends AnyVal { def toEither: Either[String, T] = self match { - case Expressions.PART.VALID(x) => Right(x) - case Expressions.PART.INVALID(x, message) => Left(s"$message: $x") + case Expressions.PART.VALID(_, _, x) => Right(x) + case Expressions.PART.INVALID(_, _, message) => Left(message) } } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala index 17771d5ac97..ac7c2fb5235 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala @@ -27,80 +27,92 @@ object Parser { private val notEndOfString = CharPred(_ != '\"') private val specialSymbols = P("\\" ~~ notEndOfString.?) - private val escapedUnicodeSymbolP = P(NoCut(unicodeSymbolP) | specialSymbols) - private val stringP: P[EXPR] = P("\"" ~/ Pass ~~ (escapedUnicodeSymbolP | notEndOfString).!.repX ~~ "\"") - .map { xs => - var errors = Vector.empty[String] - val consumedString = new StringBuilder - - xs.foreach { x => - if (x.startsWith("\\u")) { - if (x.length == 6) { - val hexCode = x.drop(2) - try { - val int = Integer.parseInt(hexCode, 16) - val unicodeSymbol = new String(Character.toChars(int)) - consumedString.append(unicodeSymbol) - } catch { - case _: NumberFormatException => - consumedString.append(x) - errors :+= s"Can't parse '$hexCode' as HEX string in '$x'" - case _: IllegalArgumentException => - consumedString.append(x) - errors :+= s"Invalid UTF-8 symbol: '$x'" + private val escapedUnicodeSymbolP: P[(Int, String, Int)] = P(Index ~~ (NoCut(unicodeSymbolP) | specialSymbols).! ~~ Index) + private val stringP: P[EXPR] = P(Index ~~ "\"" ~/ Pass ~~ (escapedUnicodeSymbolP | notEndOfString).!.repX ~~ "\"" ~~ Index) + .map { + case (start, xs, end) => + var errors = Vector.empty[String] + val consumedString = new StringBuilder + + xs.foreach { x => + if (x.startsWith("\\u")) { + if (x.length == 6) { + val hexCode = x.drop(2) + try { + val int = Integer.parseInt(hexCode, 16) + val unicodeSymbol = new String(Character.toChars(int)) + consumedString.append(unicodeSymbol) + } catch { + case _: NumberFormatException => + consumedString.append(x) + errors :+= s"can't parse '$hexCode' as HEX string in '$x'" + case _: IllegalArgumentException => + consumedString.append(x) + errors :+= s"invalid UTF-8 symbol: '$x'" + } + } else { + consumedString.append(x) + errors :+= s"incomplete UTF-8 symbol definition: '$x'" + } + } else if (x.startsWith("\\")) { + if (x.length == 2) { + consumedString.append(x(1) match { + case 'b' => "\b" + case 'f' => "\f" + case 'n' => "\n" + case 'r' => "\r" + case 't' => "\t" + case _ => + errors :+= s"""unknown escaped symbol: '$x'. The valid are \b, \f, \n, \r, \t""" + x + }) + } else { + consumedString.append(x) + errors :+= s"""invalid escaped symbol: '$x'. The valid are \b, \f, \n, \r, \t""" } } else { consumedString.append(x) - errors :+= s"Incomplete UTF-8 symbol definition: '$x'" - } - } else if (x.startsWith("\\")) { - if (x.length == 2) { - consumedString.append(x(1) match { - case 'b' => "\b" - case 'f' => "\f" - case 'n' => "\n" - case 'r' => "\r" - case 't' => "\t" - case _ => - errors :+= s"Unknown escaped symbol: '$x'" - x - }) - } else { - consumedString.append(x) - errors :+= s"Invalid escaped symbol: '$x'" } - } else { - consumedString.append(x) } - } - if (errors.isEmpty) PART.VALID(consumedString.toString) - else PART.INVALID(consumedString.toString, errors.mkString(";")) + val r = + if (errors.isEmpty) PART.VALID(start + 1, end - 1, consumedString.toString) + else PART.INVALID(start + 1, end - 1, errors.mkString(";")) + (start, end, r) } - .map(CONST_STRING(_)) + .map(Function.tupled(CONST_STRING)) - private val varName: P[PART[String]] = (char ~~ (digit | char).repX()).!.map { x => - if (keywords.contains(x)) PART.INVALID(x, "keywords are restricted") - else PART.VALID(x) + private val varName: P[PART[String]] = (Index ~~ (char ~~ (digit | char).repX()).! ~~ Index).map { + case (start, x, end) => + if (keywords.contains(x)) PART.INVALID(start, end, "keywords are restricted") + else PART.VALID(start, end, x) } - private val invalid: P[INVALID] = P(AnyChars(1).! ~ fallBackExpr.?).map { - case (xs, next) => foldInvalid(xs, next) + private val invalid: P[INVALID] = P(Index ~~ AnyChars(1).! ~~ Index ~~ fallBackExpr.?).map { + case (start, xs, end, next) => foldInvalid(start, end, xs, next) } - private def foldInvalid(xs: String, next: Option[EXPR]): INVALID = next match { - case Some(INVALID(nextXs, nextNext)) => foldInvalid(xs + nextXs, nextNext) - case x => INVALID(xs, x) + private def foldInvalid(start: Int, end: Int, xs: String, next: Option[EXPR]): INVALID = next match { + case Some(INVALID(_, endInvalid, nextXs, nextNext)) => foldInvalid(start, endInvalid, xs + nextXs, nextNext) + case x => INVALID(start, end, xs, x) } - private val numberP: P[CONST_LONG] = P(CharIn("+-").? ~ digit.repX(min = 1)).!.map(t => CONST_LONG(t.toLong)) - private val trueP: P[TRUE.type] = P("true").map(_ => TRUE) - private val falseP: P[FALSE.type] = P("false").map(_ => FALSE) - private val bracesP: P[EXPR] = P("(" ~ fallBackExpr ~ ")") - private val curlyBracesP: P[EXPR] = P("{" ~ fallBackExpr ~ "}") - private val letP: P[LET] = P("let" ~ varName ~ "=" ~ fallBackExpr).map(Function.tupled(LET(_, _, Seq.empty))) - private val refP: P[REF] = P(varName).map(REF(_)) - private val ifP: P[IF] = P("if" ~ bracesP ~ "then" ~ fallBackExpr ~ "else" ~ fallBackExpr).map { case (x, y, z) => IF(x, y, z) } + private val numberP: P[CONST_LONG] = P(Index ~~ (CharIn("+-").? ~ digit.repX(min = 1)).! ~~ Index).map { + case (start, x, end) => CONST_LONG(start, end, x.toLong) + } + private val trueP: P[TRUE] = P(Index ~~ "true".! ~~ Index).map { case (start, _, end) => TRUE(start, end) } + private val falseP: P[FALSE] = P(Index ~~ "false".! ~~ Index).map { case (start, _, end) => FALSE(start, end) } + private val bracesP: P[EXPR] = P("(" ~ fallBackExpr ~ ")") + private val curlyBracesP: P[EXPR] = P("{" ~ fallBackExpr ~ "}") + private val letP: P[LET] = P(Index ~~ "let" ~ varName ~ "=" ~ fallBackExpr ~~ Index).map { + case (start, v, e, end) => LET(start, end, v, e, Seq.empty) + } + private val refP: P[REF] = P(varName).map { x => + REF(x.start, x.end, x) + } + private val ifP: P[IF] = P(Index ~~ "if" ~ bracesP ~ "then" ~ fallBackExpr ~ "else" ~ fallBackExpr ~~ Index).map { + case (start, x, y, z, end) => IF(start, end, x, y, z) + } private val functionCallArgs: P[Seq[EXPR]] = fallBackExpr.rep(sep = ",") @@ -117,63 +129,78 @@ object Parser { val typesP: P[Seq[PART[String]]] = varName.rep(min = 1, sep = "|") val typesDefP = ( ":" ~ - (typesP | restMatchCaseInvalidP.map(x => Seq(PART.INVALID(x, "the type for variable should be specified: `case varName: Type => expr`")))) + (typesP | (Index ~~ restMatchCaseInvalidP ~~ Index).map { + case (start, x, end) => Seq(PART.INVALID(start, end, "the type for variable should be specified: `case varName: Type => expr`")) + }) ).?.map(_.getOrElse(List.empty)) P( - "case" ~/ ( + Index ~~ "case" ~/ ( (varDefP ~ typesDefP) | - restMatchCaseInvalidP.map { x => - ( - Some(PART.INVALID(x, "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), - Seq.empty[PART[String]] - ) + (Index ~~ restMatchCaseInvalidP ~~ Index).map { + case (start, x, end) => + ( + Some(PART.INVALID(start, end, "invalid syntax, should be: `case varName: Type => expr` or `case _ => expr`")), + Seq.empty[PART[String]] + ) } - ) ~ "=>" ~/ baseExpr.? + ) ~ "=>" ~/ baseExpr.? ~~ Index ).map { - case (v, types, e) => + case (start, (v, types), e, end) => + val exprStart = types.lastOption.orElse(v).fold(start)(_.end) MATCH_CASE( + start = start, + end = end, newVarName = v, types = types, - expr = e.getOrElse(INVALID("expected expression")) + expr = e.getOrElse(INVALID(exprStart, end, "expected expression")) ) } } - private lazy val matchP: P[EXPR] = P("match" ~/ fallBackExpr ~ "{" ~ NoCut(matchCaseP).rep ~ "}") + + private lazy val matchP: P[EXPR] = P(Index ~~ "match" ~/ fallBackExpr ~ "{" ~ NoCut(matchCaseP).rep ~ "}" ~~ Index) .map { - case (_, Nil) => INVALID("pattern matching requires case branches") - case (e, cases) => MATCH(e, cases.toList) + case (start, _, Nil, end) => INVALID(start, end, "pattern matching requires case branches") + case (start, e, cases, end) => MATCH(start, end, e, cases.toList) } private val accessP - : P[Accessor] = P(("." ~~ varName).map(Getter) | ("(" ~/ functionCallArgs.map(Args) ~ ")")) | ("[" ~/ fallBackExpr.map(ListIndex) ~ "]") - - private val maybeAccessP: P[EXPR] = P(extractableAtom ~~ NoCut(accessP).rep).map { - case (e, f) => - f.foldLeft(e) { (e, a) => - a match { - case Getter(n) => GETTER(e, n) - case Args(args) => - e match { - case REF(functionName) => FUNCTION_CALL(functionName, args.toList) - case _ => FUNCTION_CALL(PART.INVALID("", s"$e is not a function name"), args.toList) - } - case ListIndex(index) => FUNCTION_CALL("getElement", List(e, index)) - } + : P[Accessor] = P(("." ~/ varName).map(Getter) | ("(" ~/ functionCallArgs.map(Args) ~ ")")) | ("[" ~/ fallBackExpr.map(ListIndex) ~ "]") + + private val maybeAccessP: P[EXPR] = + P(Index ~~ extractableAtom ~~ Index ~ (Index ~~ NoCut(accessP) ~~ Index).rep) + .map { + case (start, obj, objEnd, accessors) => + accessors.foldLeft(obj) { + case (e, (accessStart, a, accessEnd)) => + a match { + case Getter(n) => GETTER(start, accessEnd, e, n) + case Args(args) => + e match { + case REF(_, _, functionName) => FUNCTION_CALL(start, accessEnd, functionName, args.toList) + case _ => FUNCTION_CALL(start, accessEnd, PART.INVALID(start, objEnd, s"'$obj' is not a function name"), args.toList) + } + case ListIndex(index) => FUNCTION_CALL(start, objEnd, PART.VALID(start, accessStart, "getElement"), List(e, index)) + } + } } - } private val byteVectorP: P[EXPR] = - P("base58'" ~/ Pass ~~ CharPred(_ != '\'').repX.! ~~ "'") - .map { xs => - val decoded = if (xs.isEmpty) Right(Array.emptyByteArray) else Global.base58Decode(xs) - decoded match { - case Left(_) => CONST_BYTEVECTOR(PART.INVALID(xs, "Can't parse Base58 string")) - case Right(r) => CONST_BYTEVECTOR(PART.VALID(ByteVector(r))) - } + P(Index ~~ "base58'" ~/ Pass ~~ CharPred(_ != '\'').repX.! ~~ "'" ~~ Index) + .map { + case (start, xs, end) => + val decoded = if (xs.isEmpty) Right(Array.emptyByteArray) else Global.base58Decode(xs) + val innerStart = start + 8 + val innerEnd = end - 1 + decoded match { + case Left(_) => CONST_BYTEVECTOR(start, end, PART.INVALID(innerStart, innerEnd, "can't parse Base58 string")) + case Right(r) => CONST_BYTEVECTOR(start, end, PART.VALID(innerStart, innerEnd, ByteVector(r))) + } } - private val block: P[EXPR] = P(letP ~ fallBackExpr).map(Function.tupled(BLOCK)) + private val block: P[EXPR] = P(Index ~~ letP ~ fallBackExpr ~~ Index).map { + case (start, l, e, end) => BLOCK(start, end, l, e) + } private val baseAtom = P(ifP | NoCut(matchP) | byteVectorP | stringP | numberP | trueP | falseP | block | maybeAccessP) private lazy val baseExpr = P(binaryOp(baseAtom, opsByPriority) | baseAtom) @@ -187,15 +214,17 @@ object Parser { case Nil => unaryOp(atom, unaryOps) case kind :: restOps => val operand = binaryOp(atom, restOps) - P(operand ~ (kind.parser.!.map(_ => kind) ~/ operand).rep()).map { - case (left: EXPR, r: Seq[(BinaryOperation, EXPR)]) => - r.foldLeft(left) { case (acc, (currKind, currOperand)) => currKind.expr(acc)(currOperand) } + P(Index ~~ operand ~ (kind.parser.!.map(_ => kind) ~ operand).rep() ~~ Index).map { + case (start, left: EXPR, r: Seq[(BinaryOperation, EXPR)], end) => + r.foldLeft(left) { case (acc, (currKind, currOperand)) => currKind.expr(start, end, acc, currOperand) } } } - def unaryOp(atom: P[EXPR], ops: Seq[(P[Any], EXPR => EXPR)]): P[EXPR] = ops.foldRight(atom) { - case ((parser, transformer), acc) => - (parser.map(_ => ()) ~ P(unaryOp(atom, ops))).map(transformer) | acc + def unaryOp(atom: P[EXPR], ops: List[UnaryOperation]): P[EXPR] = ops.foldRight(atom) { + case (op, acc) => + (Index ~~ op.parser.map(_ => ()) ~ P(unaryOp(atom, ops)) ~~ Index).map { + case (start, expr, end) => op.expr(start, end, expr) + } | acc } def apply(str: String): core.Parsed[Seq[EXPR], Char, String] = P(Start ~ fallBackExpr.rep(min = 1) ~ End).parse(str) diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala index 2e25f482b26..bcfa9e920ec 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/UnaryOperation.scala @@ -3,13 +3,30 @@ package com.wavesplatform.lang.v1.parser import com.wavesplatform.lang.v1.parser.Expressions._ import fastparse.all._ +sealed abstract class UnaryOperation { + val parser: P[Any] + def expr(start: Int, end: Int, op: EXPR): EXPR +} + object UnaryOperation { - val unaryOps = List( - P("-" ~ !CharIn('0' to '9')) -> { e: EXPR => - FUNCTION_CALL("-", List(e)) - }, - P("!") -> { e: EXPR => - FUNCTION_CALL("!", List(e)) - } + + val unaryOps: List[UnaryOperation] = List( + NEGATIVE_OP, + NOT_OP ) + + case object NEGATIVE_OP extends UnaryOperation { + override val parser: P[Any] = P("-" ~ !CharIn('0' to '9')) + override def expr(start: Int, end: Int, op: EXPR): EXPR = { + FUNCTION_CALL(start, end, PART.VALID(start, end, "-"), List(op)) + } + } + + case object NOT_OP extends UnaryOperation { + override val parser: P[Any] = P("!") + override def expr(start: Int, end: Int, op: EXPR): EXPR = { + FUNCTION_CALL(start, end, PART.VALID(start, end, "!"), List(op)) + } + } + } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index 424f0b88bf6..7c933046a00 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -3,7 +3,6 @@ package com.wavesplatform.lang.v1.testing import com.wavesplatform.lang.v1.parser.BinaryOperation import com.wavesplatform.lang.v1.parser.BinaryOperation._ import com.wavesplatform.lang.v1.parser.Expressions._ -import com.wavesplatform.lang.v1.parser.Parser.keywords import org.scalacheck._ import scodec.bits.ByteVector import scorex.crypto.encode.Base58 @@ -12,12 +11,11 @@ import scala.reflect.ClassTag trait ScriptGen { - def CONST_LONGgen: Gen[(EXPR, Long)] = Gen.choose(Long.MinValue, Long.MaxValue).map(v => (CONST_LONG(v), v)) + def CONST_LONGgen: Gen[(EXPR, Long)] = Gen.choose(Long.MinValue, Long.MaxValue).map(v => (CONST_LONG(0, 0, v), v)) def BOOLgen(gas: Int): Gen[(EXPR, Boolean)] = - if (gas > 0) - Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), NE_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1)) - else Gen.const((TRUE, true)) + if (gas > 0) Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1)) + else Gen.const((TRUE(0, 0), true)) def SUMgen(gas: Int): Gen[(EXPR, Long)] = for { @@ -25,9 +23,9 @@ trait ScriptGen { (i2, v2) <- INTGen((gas - 2) / 2) } yield if ((BigInt(v1) + BigInt(v2)).isValidLong) { - (BINARY_OP(i1, SUM_OP, i2), v1 + v2) + (BINARY_OP(0, 0, i1, SUM_OP, i2), v1 + v2) } else { - (BINARY_OP(i1, SUB_OP, i2), v1 - v2) + (BINARY_OP(0, 0, i1, SUB_OP, i2), v1 - v2) } def SUBgen(gas: Int): Gen[(EXPR, Long)] = @@ -36,9 +34,9 @@ trait ScriptGen { (i2, v2) <- INTGen((gas - 2) / 2) } yield if ((BigInt(v1) - BigInt(v2)).isValidLong) { - (BINARY_OP(i1, SUB_OP, i2), v1 - v2) + (BINARY_OP(0, 0, i1, SUB_OP, i2), v1 - v2) } else { - (BINARY_OP(i1, SUM_OP, i2), v1 + v2) + (BINARY_OP(0, 0, i1, SUM_OP, i2), v1 + v2) } def INTGen(gas: Int): Gen[(EXPR, Long)] = @@ -48,89 +46,71 @@ trait ScriptGen { SUMgen(gas - 1), SUBgen(gas - 1), IF_INTgen(gas - 1), - INTGen(gas - 1).filter(v => (-BigInt(v._2)).isValidLong).map(e => (FUNCTION_CALL("-", List(e._1)), -e._2)) + INTGen(gas - 1).filter(v => (-BigInt(v._2)).isValidLong).map(e => (FUNCTION_CALL(0, 0, PART.VALID(0, 0, "-"), List(e._1)), -e._2)) ) else CONST_LONGgen def GEgen(gas: Int): Gen[(EXPR, Boolean)] = for { - dir <- Gen.oneOf(true, false) (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - } yield - if (dir) { - (BINARY_OP(i1, GE_OP, i2), v1 >= v2) - } else { - (BINARY_OP(i2, LE_OP, i1), v1 <= v2) - } + } yield (BINARY_OP(0, 0, i1, GE_OP, i2), v1 >= v2) def GTgen(gas: Int): Gen[(EXPR, Boolean)] = for { - dir <- Gen.oneOf(true, false) (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - } yield - if (dir) { - (BINARY_OP(i1, GT_OP, i2), v1 > v2) - } else { - (BINARY_OP(i2, LT_OP, i1), v1 < v2) - } + } yield (BINARY_OP(0, 0, i1, GT_OP, i2), v1 > v2) def EQ_INTgen(gas: Int): Gen[(EXPR, Boolean)] = for { (i1, v1) <- INTGen((gas - 2) / 2) (i2, v2) <- INTGen((gas - 2) / 2) - } yield (BINARY_OP(i1, EQ_OP, i2), v1 == v2) - - def NE_INTgen(gas: Int): Gen[(EXPR, Boolean)] = - for { - (i1, v1) <- INTGen((gas - 2) / 2) - (i2, v2) <- INTGen((gas - 2) / 2) - } yield (BINARY_OP(i1, NE_OP, i2), v1 != v2) + } yield (BINARY_OP(0, 0, i1, EQ_OP, i2), v1 == v2) def ANDgen(gas: Int): Gen[(EXPR, Boolean)] = for { (i1, v1) <- BOOLgen((gas - 2) / 2) (i2, v2) <- BOOLgen((gas - 2) / 2) - } yield (BINARY_OP(i1, AND_OP, i2), v1 && v2) + } yield (BINARY_OP(0, 0, i1, AND_OP, i2), v1 && v2) def ORgen(gas: Int): Gen[(EXPR, Boolean)] = for { (i1, v1) <- BOOLgen((gas - 2) / 2) (i2, v2) <- BOOLgen((gas - 2) / 2) - } yield (BINARY_OP(i1, OR_OP, i2), v1 || v2) + } yield (BINARY_OP(0, 0, i1, OR_OP, i2), v1 || v2) def IF_BOOLgen(gas: Int): Gen[(EXPR, Boolean)] = for { (cnd, vcnd) <- BOOLgen((gas - 3) / 3) (t, vt) <- BOOLgen((gas - 3) / 3) (f, vf) <- BOOLgen((gas - 3) / 3) - } yield (IF(cnd, t, f), if (vcnd) vt else vf) + } yield (IF(0, 0, cnd, t, f), if (vcnd) { vt } else { vf }) def IF_INTgen(gas: Int): Gen[(EXPR, Long)] = for { (cnd, vcnd) <- BOOLgen((gas - 3) / 3) (t, vt) <- INTGen((gas - 3) / 3) (f, vf) <- INTGen((gas - 3) / 3) - } yield (IF(cnd, t, f), if (vcnd) vt else vf) + } yield (IF(0, 0, cnd, t, f), if (vcnd) { vt } else { vf }) def STRgen: Gen[EXPR] = - Gen.identifier.map(PART.VALID[String]).map(CONST_STRING(_)) + Gen.identifier.map(PART.VALID[String](0, 0, _)).map(CONST_STRING(0, 0, _)) def LETgen(gas: Int): Gen[LET] = for { name <- Gen.identifier (value, _) <- BOOLgen((gas - 3) / 3) - } yield LET(PART.VALID(name), value, Seq.empty) + } yield LET(0, 0, PART.VALID(0, 0, name), value, Seq.empty) def REFgen: Gen[EXPR] = - Gen.identifier.filter(!keywords(_)).map(PART.VALID[String]).map(REF(_)) + Gen.identifier.map(PART.VALID[String](0, 0, _)).map(REF(0, 0, _)) def BLOCKgen(gas: Int): Gen[EXPR] = for { let <- LETgen((gas - 3) / 3) body <- Gen.oneOf(BOOLgen((gas - 3) / 3).map(_._1), BLOCKgen((gas - 3) / 3)) // BLOCKGen wasn't add to BOOLGen since issue: NODE-700 - } yield BLOCK(let, body) + } yield BLOCK(0, 0, let, body) private val spaceChars: Seq[Char] = " \t\n\r" @@ -144,46 +124,33 @@ trait ScriptGen { for { pred <- whitespaces post <- whitespaces - } yield s" $expr " //pred + expr + post + } yield pred + expr + post private def toString[T](part: PART[T])(implicit ct: ClassTag[T]): String = part match { - case PART.VALID(x: String) => x - case PART.VALID(xs: ByteVector) => Base58.encode(xs.toArray) - case PART.INVALID(consumed, _) => consumed - case _ => throw new RuntimeException(s"Can't stringify $part") + case PART.VALID(_, _, x: String) => x + case PART.VALID(_, _, xs: ByteVector) => Base58.encode(xs.toArray) + case _ => throw new RuntimeException(s"Can't stringify $part") } def toString(expr: EXPR): Gen[String] = expr match { - case CONST_LONG(x) => withWhitespaces(s"$x") - case REF(x) => withWhitespaces(toString(x)) - case CONST_STRING(x) => withWhitespaces(s"""\"${toString(x)}\"""") - case CONST_BYTEVECTOR(x) => withWhitespaces(s"""base58'${toString(x)}'""") - case TRUE => withWhitespaces("true") - case FALSE => withWhitespaces("false") - case FUNCTION_CALL(PART.VALID("-"), List(CONST_LONG(v))) if v >= 0 => s"-($v)" - case FUNCTION_CALL(op, List(e)) => toString(e).map(e => s"${toString(op)}$e") - case BINARY_OP(x, LE_OP, y) => - for { - arg2 <- toString(x) - arg1 <- toString(y) - } yield s"($arg1<=$arg2)" - case BINARY_OP(x, LT_OP, y) => - for { - arg2 <- toString(x) - arg1 <- toString(y) - } yield s"($arg1<$arg2)" - case BINARY_OP(x, op: BinaryOperation, y) => + case CONST_LONG(_, _, x) => withWhitespaces(s"$x") + case REF(_, _, x) => withWhitespaces(toString(x)) + case CONST_STRING(_, _, x) => withWhitespaces(s"""\"${toString(x)}\"""") + case CONST_BYTEVECTOR(_, _, x) => withWhitespaces(s"""base58'${toString(x)}'""") + case _: TRUE => withWhitespaces("true") + case _: FALSE => withWhitespaces("false") + case BINARY_OP(_, _, x, op: BinaryOperation, y) => for { arg1 <- toString(x) arg2 <- toString(y) } yield s"($arg1${opsToFunctions(op)}$arg2)" - case IF(cond, x, y) => + case IF(_, _, cond, x, y) => for { c <- toString(cond) t <- toString(x) f <- toString(y) } yield s"(if ($c) then $t else $f)" - case BLOCK(let, body) => + case BLOCK(_, _, let, body) => for { v <- toString(let.value) b <- toString(body) @@ -195,17 +162,8 @@ trait ScriptGen { trait ScriptGenParser extends ScriptGen { override def BOOLgen(gas: Int): Gen[(EXPR, Boolean)] = { if (gas > 0) - Gen.oneOf( - GEgen(gas - 1), - GTgen(gas - 1), - EQ_INTgen(gas - 1), - ANDgen(gas - 1), - ORgen(gas - 1), - IF_BOOLgen(gas - 1), - REFgen.map(r => (r, false)), - BOOLgen(gas - 1).map(e => (FUNCTION_CALL("!", List(e._1)), !e._2)) - ) - else Gen.const((TRUE, true)) + Gen.oneOf(GEgen(gas - 1), GTgen(gas - 1), EQ_INTgen(gas - 1), ANDgen(gas - 1), ORgen(gas - 1), IF_BOOLgen(gas - 1), REFgen.map(r => (r, false))) + else Gen.const((TRUE(0, 0), true)) } override def INTGen(gas: Int): Gen[(EXPR, Long)] = diff --git a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OracleDataTest.scala b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OracleDataTest.scala index afa59ca77b1..10867dc7164 100644 --- a/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OracleDataTest.scala +++ b/src/test/scala/com/wavesplatform/state/diffs/smart/scenarios/OracleDataTest.scala @@ -30,18 +30,12 @@ class OracleDataTest extends PropSpec with PropertyChecks with Matchers with Tra bin <- binaryEntryGen(500, dataAsciiKeyGen).filter(e => e.key != long.key && e.key != bool.key) str <- stringEntryGen(500, dataAsciiKeyGen).filter(e => e.key != long.key && e.key != bool.key && e.key != bin.key) dataTransaction <- dataTransactionGenP(oracle, List(long, bool, bin, str)) - allFieldsRequiredScript = s""" - | - | let oracle = extract(addressFromString("${oracle.address}")) - | let long = extract(getLong(oracle,"${long.key}")) == ${long.value} - | let bool = extract(getBoolean(oracle,"${bool.key}")) == ${bool.value} - | let bin = extract(getByteArray(oracle,"${bin.key}")) == base58'${bin.value.base58}' - | let str = extract(getString(oracle,"${str.key}")) == "${str.value}" - | long && bool && bin && str - | - | - | - """.stripMargin + allFieldsRequiredScript = s"""let oracle = extract(addressFromString("${oracle.address}")) + |let long = extract(getLong(oracle,"${long.key}")) == ${long.value} + |let bool = extract(getBoolean(oracle,"${bool.key}")) == ${bool.value} + |let bin = extract(getByteArray(oracle,"${bin.key}")) == base58'${bin.value.base58}' + |let str = extract(getString(oracle,"${str.key}")) == "${str.value}" + |long && bool && bin && str""".stripMargin setScript <- { val untypedAllFieldsRequiredScript = Parser(allFieldsRequiredScript).get.value assert(untypedAllFieldsRequiredScript.size == 1) From 568de1d5785deb62ccdaff56a87710fde9329f63 Mon Sep 17 00:00:00 2001 From: Vyatcheslav Suharnikov Date: Fri, 25 May 2018 11:39:07 +0300 Subject: [PATCH 46/52] NODE-745 * Parser fixes: * NODE-700: ; and \n rules in BLOCK; * Better accessors parsing: now it treats multiline function calls as invalid: (foo)\n(bar, baz); * Better numbers parsing; * NoCut for LET; * ParserTest: additional tests for BLOCK (NODE-700); * ParserTest: more detailed information about failure. --- .../com/wavesplatform/lang/ParserTest.scala | 127 ++++++++++++++++-- .../wavesplatform/lang/v1/parser/Parser.scala | 46 +++++-- .../lang/v1/testing/ScriptGen.scala | 8 +- 3 files changed, 154 insertions(+), 27 deletions(-) diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala index 3285f7335cd..69e5dae901c 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala @@ -21,21 +21,52 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG println(s"Can't parse (len=${x.length}): \n$x\n") throw new TestFailedException(s"Expected 1 expression, but got ${r.size}: $r", 0) } else r.head - case e @ Failure(_, i, _) => - println( - s"Can't parse (len=${x.length}): \n$x\n\nError: $e\nPosition ($i): '${x.slice(i, i + 1)}'\nTraced:\n${e.extra.traced.fullStack - .mkString("\n")}") - throw new TestFailedException("Test failed", 0) + case e: Failure[Char, String] => catchParseError(x, e) } private def parseAll(x: String): Seq[EXPR] = Parser(x) match { - case Success(r, _) => r - case e @ Failure(_, i, _) => - println(x) - println( - s"Can't parse (len=${x.length}): \n$x\n\nError: $e\nPosition ($i): '${x.slice(i, i + 1)}'\nTraced:\n${e.extra.traced.fullStack - .mkString("\n")}") - throw new TestFailedException("Test failed", 0) + case Success(r, _) => r + case e: Failure[Char, String] => catchParseError(x, e) + } + + private def catchParseError(x: String, e: Failure[Char, String]): Nothing = { + import e.{index => i} + println(s"val codeInBytes = new String(Array[Byte](${x.getBytes.mkString(",")}))") + println(s"""val codeInStr = "${escapedCode(x)}"""") + println(s"Can't parse (len=${x.length}): \n$x\n\nError: $e\nPosition ($i): '${x.slice(i, i + 1)}'\nTraced:\n${e.extra.traced.fullStack + .mkString("\n")}") + throw new TestFailedException("Test failed", 0) + } + + private def escapedCode(s: String): String = + s.flatMap { + case '"' => "\\\"" + case '\n' => "\\n" + case '\r' => "\\r" + case '\t' => "\\t" + case x => x.toChar.toString + }.mkString + + private def cleanOffsets(l: LET): LET = + l.copy(start = 0, end = 0, name = cleanOffsets(l.name), value = cleanOffsets(l.value), types = l.types.map(cleanOffsets(_))) + + private def cleanOffsets[T](p: PART[T]): PART[T] = p match { + case PART.VALID(_, _, x) => PART.VALID(0, 0, x) + case PART.INVALID(_, _, x) => PART.INVALID(0, 0, x) + } + + private def cleanOffsets(expr: EXPR): EXPR = expr match { + case x: CONST_LONG => x.copy(start = 0, end = 0) + case x: REF => x.copy(start = 0, end = 0, key = cleanOffsets(x.key)) + case x: CONST_STRING => x.copy(start = 0, end = 0, value = cleanOffsets(x.value)) + case x: CONST_BYTEVECTOR => x.copy(start = 0, end = 0, value = cleanOffsets(x.value)) + case x: TRUE => x.copy(start = 0, end = 0) + case x: FALSE => x.copy(start = 0, end = 0) + case x: BINARY_OP => x.copy(start = 0, end = 0, a = cleanOffsets(x.a), b = cleanOffsets(x.b)) + case x: IF => x.copy(start = 0, end = 0, cond = cleanOffsets(x.cond), ifTrue = cleanOffsets(x.ifTrue), ifFalse = cleanOffsets(x.ifFalse)) + case x: BLOCK => x.copy(start = 0, end = 0, let = cleanOffsets(x.let), body = cleanOffsets(x.body)) + case x: FUNCTION_CALL => x.copy(start = 0, end = 0, name = cleanOffsets(x.name), args = x.args.map(cleanOffsets(_))) + case _ => ??? } private def genElementCheck(gen: Gen[EXPR]): Unit = { @@ -47,7 +78,7 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG forAll(testGen) { case (expr, str) => withClue(str) { - parseOne(str) shouldBe expr + cleanOffsets(parseOne(str)) shouldBe expr } } } @@ -175,6 +206,62 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG parseOne("\"foo \\\"") shouldBe CONST_STRING(0, 7, PART.INVALID(1, 6, "invalid escaped symbol: '\\'. The valid are \b, \f, \n, \r, \t")) } + property("block: multiline without ;") { + val s = + """let q = 1 + |c""".stripMargin + parseOne(s) shouldBe BLOCK( + 0, + 11, + LET(0, 9, PART.VALID(4, 5, "q"), CONST_LONG(8, 9, 1), List.empty), + REF(10, 11, PART.VALID(10, 11, "c")) + ) + } + + property("block: multiline with ; at end of let") { + val s = + """let q = 1; + |c""".stripMargin + parseOne(s) shouldBe BLOCK( + 0, + 12, + LET(0, 9, PART.VALID(4, 5, "q"), CONST_LONG(8, 9, 1), List.empty), + REF(11, 12, PART.VALID(11, 12, "c")) + ) + } + + property("block: multiline with ; at start of body") { + val s = + """let q = 1 + |; c""".stripMargin + parseOne(s) shouldBe BLOCK( + 0, + 13, + LET(0, 9, PART.VALID(4, 5, "q"), CONST_LONG(8, 9, 1), List.empty), + REF(12, 13, PART.VALID(12, 13, "c")) + ) + } + + property("block: oneline") { + val s = "let q = 1; c" + parseOne(s) shouldBe BLOCK( + 0, + 12, + LET(0, 9, PART.VALID(4, 5, "q"), CONST_LONG(8, 9, 1), List.empty), + REF(11, 12, PART.VALID(11, 12, "c")) + ) + } + + property("block: invalid") { + val s = "let q = 1 c" + parseOne(s) shouldBe BLOCK( + 0, + 11, + LET(0, 9, PART.VALID(4, 5, "q"), CONST_LONG(8, 9, 1), List.empty), + INVALID(9, 9, "can't find a separator. Did you mean ';' or '\\n' ?", Some(REF(10, 11, PART.VALID(10, 11, "c")))) + ) + } + property("reserved keywords are invalid variable names in block: if") { val script = s"""let if = 1 @@ -383,7 +470,19 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG ) } - // multiple getters + property("braces in block's let and body") { + val text = + """let a = (foo) + |(bar)""".stripMargin + parseOne(text) shouldBe BLOCK( + 0, + 19, + LET(0, 13, PART.VALID(4, 5, "a"), REF(9, 12, PART.VALID(9, 12, "foo")), List.empty), + REF(15, 18, PART.VALID(15, 18, "bar")) + ) + } + + // @TODO multiple getters test property("crypto functions: sha256") { val text = "❤✓☀★☂♞☯☭☢€☎∞❄♫\u20BD=test message" diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala index ac7c2fb5235..d3ac9245401 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala @@ -8,11 +8,12 @@ import scodec.bits.ByteVector object Parser { - private val Global = com.wavesplatform.lang.hacks.Global // Hack for IDEA + private val Global = com.wavesplatform.lang.hacks.Global // Hack for IDEA + private val Whitespaces: Set[Char] = " \t\r\n".toSet private val White = WhitespaceApi.Wrapper { import fastparse.all._ - NoTrace(CharIn(" ", "\t", "\r", "\n").rep) + NoTrace(CharIn(Whitespaces.toSeq).rep) } import White._ @@ -97,14 +98,14 @@ object Parser { case x => INVALID(start, end, xs, x) } - private val numberP: P[CONST_LONG] = P(Index ~~ (CharIn("+-").? ~ digit.repX(min = 1)).! ~~ Index).map { + private val numberP: P[CONST_LONG] = P(Index ~~ (CharIn("+-").? ~~ digit.repX(min = 1)).! ~~ Index).map { case (start, x, end) => CONST_LONG(start, end, x.toLong) } private val trueP: P[TRUE] = P(Index ~~ "true".! ~~ Index).map { case (start, _, end) => TRUE(start, end) } private val falseP: P[FALSE] = P(Index ~~ "false".! ~~ Index).map { case (start, _, end) => FALSE(start, end) } private val bracesP: P[EXPR] = P("(" ~ fallBackExpr ~ ")") private val curlyBracesP: P[EXPR] = P("{" ~ fallBackExpr ~ "}") - private val letP: P[LET] = P(Index ~~ "let" ~ varName ~ "=" ~ fallBackExpr ~~ Index).map { + private val letP: P[LET] = P(Index ~~ "let" ~ varName ~ "=" ~/ fallBackExpr ~~ Index).map { case (start, v, e, end) => LET(start, end, v, e, Seq.empty) } private val refP: P[REF] = P(varName).map { x => @@ -164,11 +165,14 @@ object Parser { case (start, e, cases, end) => MATCH(start, end, e, cases.toList) } - private val accessP - : P[Accessor] = P(("." ~/ varName).map(Getter) | ("(" ~/ functionCallArgs.map(Args) ~ ")")) | ("[" ~/ fallBackExpr.map(ListIndex) ~ "]") + private val accessP: P[(Int, Accessor, Int)] = P( + ("" ~ Index ~ "." ~/ varName.map(Getter) ~~ Index) | + (Index ~~ "(" ~/ functionCallArgs.map(Args) ~ ")" ~~ Index) | + (Index ~~ "[" ~/ fallBackExpr.map(ListIndex) ~ "]" ~~ Index) + ) private val maybeAccessP: P[EXPR] = - P(Index ~~ extractableAtom ~~ Index ~ (Index ~~ NoCut(accessP) ~~ Index).rep) + P(Index ~~ extractableAtom ~~ Index ~~ NoCut(accessP).rep) .map { case (start, obj, objEnd, accessors) => accessors.foldLeft(obj) { @@ -198,8 +202,30 @@ object Parser { } } - private val block: P[EXPR] = P(Index ~~ letP ~ fallBackExpr ~~ Index).map { - case (start, l, e, end) => BLOCK(start, end, l, e) + private val block: P[EXPR] = { + // Hack to force parse of "\n". Otherwise it is treated as a separator + val newLineSep = { + val rawSep = '\n' + val white = WhitespaceApi.Wrapper { + import fastparse.all._ + NoTrace(CharIn((Whitespaces - rawSep).toSeq).rep) + } + + import white._ + P("" ~ rawSep.toString.rep(min = 1)) + } + + P( + Index ~~ + letP ~~ + Index ~~ (("" ~ ";").!.map(_ => true) | newLineSep.!.map(_ => true) | "".!.map(_ => false)) ~~ Index ~ + fallBackExpr ~~ + Index) + .map { + case (start, l, sepStart, sep, sepEnd, e, end) => + if (sep) BLOCK(start, end, l, e) + else BLOCK(start, end, l, INVALID(sepStart, sepEnd, "can't find a separator. Did you mean ';' or '\\n' ?", Some(e))) + } } private val baseAtom = P(ifP | NoCut(matchP) | byteVectorP | stringP | numberP | trueP | falseP | block | maybeAccessP) @@ -227,5 +253,5 @@ object Parser { } | acc } - def apply(str: String): core.Parsed[Seq[EXPR], Char, String] = P(Start ~ fallBackExpr.rep(min = 1) ~ End).parse(str) + def apply(str: String): core.Parsed[Seq[EXPR], Char, String] = P(Start ~ (!End ~ fallBackExpr).rep(min = 1)).parse(str) } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index 7c933046a00..993e892810d 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -152,9 +152,11 @@ trait ScriptGen { } yield s"(if ($c) then $t else $f)" case BLOCK(_, _, let, body) => for { - v <- toString(let.value) - b <- toString(body) - } yield s"let ${toString(let.name)} = $v $b\n" + v <- toString(let.value) + b <- toString(body) + isNewLine <- Arbitrary.arbBool.arbitrary + sep <- if (isNewLine) Gen.const("\n") else withWhitespaces(";") + } yield s"let ${toString(let.name)} = $v$sep$b" case _ => ??? } } From ad124405141939ea08cca8d94a969891edf03de9 Mon Sep 17 00:00:00 2001 From: Vyatcheslav Suharnikov Date: Fri, 25 May 2018 13:09:07 +0300 Subject: [PATCH 47/52] NODE-745 Fixed CompilerV1Test, IntegrationTest --- .../wavesplatform/lang/IntegrationTest.scala | 12 ++++--- .../com/wavesplatform/lang/ParserTest.scala | 2 +- .../lang/typechecker/CompilerV1Test.scala | 34 +++++++++---------- .../lang/v1/compiler/CompilerV1.scala | 2 +- .../lang/v1/parser/Expressions.scala | 2 +- .../lang/v1/testing/ScriptGen.scala | 7 +++- 6 files changed, 34 insertions(+), 25 deletions(-) diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala index 455777b67a9..42ff2ee37ba 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala @@ -79,8 +79,8 @@ class IntegrationTest extends PropSpec with PropertyChecks with ScriptGen with M } property("equals some lang structure") { - eval[Boolean]("let x = (-7763390488025868909>-1171895536391400041) let v = false (v&&true)") shouldBe Right(false) - eval[Boolean]("let mshUmcl = (if(true) then true else true) true || mshUmcl") shouldBe Right(true) + eval[Boolean]("let x = (-7763390488025868909>-1171895536391400041); let v = false; (v&&true)") shouldBe Right(false) + eval[Boolean]("let mshUmcl = (if(true) then true else true); true || mshUmcl") shouldBe Right(true) eval[Long]("""if(((1+-1)==-1)) then 1 else (1+1)""") shouldBe Right(2) eval[Boolean]("""((((if(true) then 1 else 1)==2)||((if(true) |then true else true)&&(true||true)))||(if(((1>1)||(-1>=-1))) @@ -94,7 +94,9 @@ class IntegrationTest extends PropSpec with PropertyChecks with ScriptGen with M str <- toString(expr) } yield (str, res)) { case (str, res) => - eval[Long](str) shouldBe Right(res) + withClue(str) { + eval[Long](str) shouldBe Right(res) + } }) forAll(for { @@ -102,7 +104,9 @@ class IntegrationTest extends PropSpec with PropertyChecks with ScriptGen with M str <- toString(expr) } yield (str, res)) { case (str, res) => - eval[Boolean](str) shouldBe Right(res) + withClue(str) { + eval[Boolean](str) shouldBe Right(res) + } } } diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala index 69e5dae901c..f6f71f3b2da 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala @@ -66,7 +66,7 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG case x: IF => x.copy(start = 0, end = 0, cond = cleanOffsets(x.cond), ifTrue = cleanOffsets(x.ifTrue), ifFalse = cleanOffsets(x.ifFalse)) case x: BLOCK => x.copy(start = 0, end = 0, let = cleanOffsets(x.let), body = cleanOffsets(x.body)) case x: FUNCTION_CALL => x.copy(start = 0, end = 0, name = cleanOffsets(x.name), args = x.args.map(cleanOffsets(_))) - case _ => ??? + case _ => throw new NotImplementedError(s"toString for ${expr.getClass.getSimpleName}") } private def genElementCheck(gen: Gen[EXPR]): Unit = { diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala index a47385557fd..b9c7bdbd1ba 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/typechecker/CompilerV1Test.scala @@ -30,7 +30,7 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr v.tpe shouldBe LONG } - treeTypeTest(s"unitOnNone(NONE)")( + treeTypeTest("unitOnNone(NONE)")( ctx = typeCheckerContext, expr = Expressions.FUNCTION_CALL(0, 0, @@ -90,7 +90,7 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr expectedResult = Right(FUNCTION_CALL(multiplierFunction.header, List(CONST_LONG(1), CONST_LONG(2)), LONG)) ) - treeTypeTest(s"idOptionLong(NONE)")( + treeTypeTest("idOptionLong(NONE)")( ctx = typeCheckerContext, expr = Expressions.FUNCTION_CALL( 0, @@ -101,7 +101,7 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr expectedResult = Right(FUNCTION_CALL(idOptionLong.header, List(REF("None", OPTION(NOTHING))), UNIT)) ) - treeTypeTest(s"idOptionLong(SOME(NONE))")( + treeTypeTest("idOptionLong(SOME(NONE))")( ctx = typeCheckerContext, expr = Expressions.FUNCTION_CALL( 0, @@ -112,7 +112,7 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr 0, 0, Expressions.PART.VALID(0, 0, "Some"), - List(Expressions.REF(0, 0, Expressions.PART.INVALID(0, 0, "None"))) + List(Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "None"))) ) ) ), @@ -120,7 +120,7 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr Right(FUNCTION_CALL(idOptionLong.header, List(FUNCTION_CALL(some.header, List(REF("None", OPTION(NOTHING))), OPTION(OPTION(NOTHING)))), UNIT)) ) - treeTypeTest(s"idOptionLong(SOME(CONST_LONG(3)))")( + treeTypeTest("idOptionLong(SOME(CONST_LONG(3)))")( ctx = typeCheckerContext, expr = Expressions.FUNCTION_CALL( 0, @@ -149,40 +149,40 @@ class CompilerV1Test extends PropSpec with PropertyChecks with Matchers with Scr expr = Expressions.BLOCK( 0, 0, - Expressions.LET(0, 0, Expressions.PART.INVALID(0, 0, "it is invalid!"), Expressions.TRUE(0, 0), Seq.empty), + Expressions.LET(0, 0, Expressions.PART.INVALID(0, 1, "can't parse"), Expressions.TRUE(0, 0), Seq.empty), Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "x")) ), - expectedResult = Left("Typecheck failed: it is invalid!: ###") + expectedResult = Left("Typecheck failed: Can't compile an invalid instruction: can't parse in 0-1") ) treeTypeTest("Invalid GETTER")( ctx = typeCheckerContext, - expr = Expressions.GETTER(0, 0, Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "x")), Expressions.PART.INVALID(0, 0, "it is invalid!")), - expectedResult = Left("Typecheck failed: it is invalid!: ###") + expr = Expressions.GETTER(0, 0, Expressions.REF(0, 0, Expressions.PART.VALID(0, 0, "x")), Expressions.PART.INVALID(2, 3, "can't parse")), + expectedResult = Left("Typecheck failed: Can't compile an invalid instruction: can't parse in 2-3") ) treeTypeTest("Invalid BYTEVECTOR")( ctx = typeCheckerContext, - expr = Expressions.CONST_BYTEVECTOR(0, 0, Expressions.PART.INVALID(0, 0, "it is invalid!")), - expectedResult = Left("Typecheck failed: it is invalid!: foo") + expr = Expressions.CONST_BYTEVECTOR(0, 0, Expressions.PART.INVALID(0, 0, "can't parse")), + expectedResult = Left("Typecheck failed: can't parse in 0-0") ) treeTypeTest("Invalid STRING")( ctx = typeCheckerContext, - expr = Expressions.CONST_STRING(0, 0, Expressions.PART.INVALID(0, 0, "it is invalid!")), - expectedResult = Left("Typecheck failed: it is invalid!: \\u1") + expr = Expressions.CONST_STRING(0, 0, Expressions.PART.INVALID(0, 0, "can't parse")), + expectedResult = Left("Typecheck failed: can't parse in 0-0") ) treeTypeTest("Invalid REF")( ctx = typeCheckerContext, - expr = Expressions.REF(0, 0, Expressions.PART.INVALID(0, 0, "it is invalid!")), - expectedResult = Left("Typecheck failed: it is invalid!: ###") + expr = Expressions.REF(0, 0, Expressions.PART.INVALID(0, 0, "can't parse")), + expectedResult = Left("Typecheck failed: Can't compile an invalid instruction: can't parse in 0-0") ) treeTypeTest("Invalid FUNCTION_CALL")( ctx = typeCheckerContext, - expr = Expressions.FUNCTION_CALL(0, 0, Expressions.PART.INVALID(0, 0, "it is invalid!"), List.empty), - expectedResult = Left("Typecheck failed: it is invalid!: ###") + expr = Expressions.FUNCTION_CALL(0, 0, Expressions.PART.INVALID(0, 0, "can't parse"), List.empty), + expectedResult = Left("Typecheck failed: Can't compile an invalid instruction: can't parse in 0-0") ) treeTypeTest("INVALID")( diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala index 152d5f42282..08007270772 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/CompilerV1.scala @@ -264,7 +264,7 @@ object CompilerV1 { private def handlePart[T](part: PART[T])(f: T => EXPR): SetTypeResult[EXPR] = part match { case PART.VALID(_, _, x) => EitherT.pure(f(x)) - case PART.INVALID(start, end, message) => EitherT.leftT[Coeval, EXPR](s"$message at $start-$end") + case PART.INVALID(start, end, message) => EitherT.leftT[Coeval, EXPR](s"$message in $start-$end") } def apply(c: CompilerContext, expr: Expressions.EXPR): CompilationResult[EXPR] = { diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala index 9e02e03d853..d6060a0df06 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Expressions.scala @@ -39,7 +39,7 @@ object Expressions { implicit class PartOps[T](val self: PART[T]) extends AnyVal { def toEither: Either[String, T] = self match { case Expressions.PART.VALID(_, _, x) => Right(x) - case Expressions.PART.INVALID(_, _, message) => Left(message) + case Expressions.PART.INVALID(s, e, message) => Left(s"Can't compile an invalid instruction: $message in $s-$e") } } diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala index 993e892810d..5aa425be968 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala @@ -157,7 +157,12 @@ trait ScriptGen { isNewLine <- Arbitrary.arbBool.arbitrary sep <- if (isNewLine) Gen.const("\n") else withWhitespaces(";") } yield s"let ${toString(let.name)} = $v$sep$b" - case _ => ??? + + case FUNCTION_CALL(_, _, PART.VALID(_, _, "-"), List(CONST_LONG(_, _, v))) if v >= 0 => + s"-($v)" + case FUNCTION_CALL(_, _, op, List(e)) => toString(e).map(e => s"${toString(op)}$e") + + case x => throw new NotImplementedError(s"toString for ${x.getClass.getSimpleName}") } } From 49d4d508607d9fa1ae14379c60ae6c0843f1de42 Mon Sep 17 00:00:00 2001 From: Vyatcheslav Suharnikov Date: Fri, 25 May 2018 13:24:59 +0300 Subject: [PATCH 48/52] NODE-745 Tests for multiple accessors in ParserTest --- .../com/wavesplatform/lang/ParserTest.scala | 55 ++++++++++++++++++- .../wavesplatform/lang/v1/parser/Parser.scala | 2 +- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala index f6f71f3b2da..e6919a2dd87 100644 --- a/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala +++ b/lang/jvm/src/test/scala/com/wavesplatform/lang/ParserTest.scala @@ -470,6 +470,59 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG ) } + property("multiple getters") { + parseOne("x.y.z") shouldBe GETTER(0, 5, GETTER(0, 3, REF(0, 1, PART.VALID(0, 1, "x")), PART.VALID(2, 3, "y")), PART.VALID(4, 5, "z")) + } + + property("array accessor") { + parseOne("x[0]") shouldBe FUNCTION_CALL(0, 4, PART.VALID(1, 4, "getElement"), List(REF(0, 1, PART.VALID(0, 1, "x")), CONST_LONG(2, 3, 0))) + } + + property("multiple array accessors") { + parseOne("x[0][1]") shouldBe FUNCTION_CALL( + 0, + 7, + PART.VALID(4, 7, "getElement"), + List( + FUNCTION_CALL(0, 4, PART.VALID(1, 4, "getElement"), List(REF(0, 1, PART.VALID(0, 1, "x")), CONST_LONG(2, 3, 0))), + CONST_LONG(5, 6, 1) + ) + ) + } + + property("accessor and getter") { + parseOne("x[0].y") shouldBe GETTER( + 0, + 6, + FUNCTION_CALL(0, 4, PART.VALID(1, 4, "getElement"), List(REF(0, 1, PART.VALID(0, 1, "x")), CONST_LONG(2, 3, 0))), + PART.VALID(5, 6, "y") + ) + } + + property("getter and accessor") { + parseOne("x.y[0]") shouldBe FUNCTION_CALL( + 0, + 6, + PART.VALID(3, 6, "getElement"), + List( + GETTER(0, 3, REF(0, 1, PART.VALID(0, 1, "x")), PART.VALID(2, 3, "y")), + CONST_LONG(4, 5, 0) + ) + ) + } + + property("function call and accessor") { + parseOne("x(y)[0]") shouldBe FUNCTION_CALL( + 0, + 7, + PART.VALID(4, 7, "getElement"), + List( + FUNCTION_CALL(0, 4, PART.VALID(0, 1, "x"), List(REF(2, 3, PART.VALID(2, 3, "y")))), + CONST_LONG(5, 6, 0) + ) + ) + } + property("braces in block's let and body") { val text = """let a = (foo) @@ -482,8 +535,6 @@ class ParserTest extends PropSpec with PropertyChecks with Matchers with ScriptG ) } - // @TODO multiple getters test - property("crypto functions: sha256") { val text = "❤✓☀★☂♞☯☭☢€☎∞❄♫\u20BD=test message" val encodedText = ScorexBase58.encode(text.getBytes) diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala index d3ac9245401..9f25b6c6515 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala @@ -184,7 +184,7 @@ object Parser { case REF(_, _, functionName) => FUNCTION_CALL(start, accessEnd, functionName, args.toList) case _ => FUNCTION_CALL(start, accessEnd, PART.INVALID(start, objEnd, s"'$obj' is not a function name"), args.toList) } - case ListIndex(index) => FUNCTION_CALL(start, objEnd, PART.VALID(start, accessStart, "getElement"), List(e, index)) + case ListIndex(index) => FUNCTION_CALL(start, accessEnd, PART.VALID(accessStart, accessEnd, "getElement"), List(e, index)) } } } From 9f4efd1e76bd786594dfce6eb08866cec3a37a7d Mon Sep 17 00:00:00 2001 From: peterz Date: Fri, 25 May 2018 14:36:10 +0300 Subject: [PATCH 49/52] NODE-773 API incompatibilities in v0.13.0 --- src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala | 4 ++-- .../scala/scorex/transaction/assets/IssueTransaction.scala | 1 + .../scorex/transaction/transfer/TransferTransaction.scala | 1 + src/test/scala/scorex/api/http/AssetsApiRouteSpec.scala | 4 ++-- .../scorex/transaction/IssueTransactionV1Specification.scala | 3 ++- .../scorex/transaction/IssueTransactionV2Specification.scala | 3 ++- .../transaction/TransferTransactionV1Specification.scala | 3 ++- .../transaction/TransferTransactionV2Specification.scala | 3 ++- 8 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala b/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala index d09f47c62d9..0e098714906 100755 --- a/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala +++ b/src/main/scala/scorex/api/http/assets/AssetsApiRoute.scala @@ -273,8 +273,8 @@ case class AssetsApiRoute(settings: RestAPISettings, wallet: Wallet, utx: UtxPoo "decimals" -> JsNumber(tx.decimals.toInt), "reissuable" -> JsBoolean(description.reissuable), "quantity" -> JsNumber(BigDecimal(description.totalVolume)), - "script" -> JsString(description.script.fold("")(_.bytes().base58)), - "scriptText" -> JsString(description.script.fold("")(_.text)), + "script" -> Json.toJson(description.script.map(_.bytes().base58)), + "scriptText" -> Json.toJson(description.script.map(_.text)), "complexity" -> JsNumber(complexity), "extraFee" -> JsNumber(if (description.script.isEmpty) 0 else CommonValidation.ScriptExtraFee), "minSponsoredAssetFee" -> (description.sponsorship match { diff --git a/src/main/scala/scorex/transaction/assets/IssueTransaction.scala b/src/main/scala/scorex/transaction/assets/IssueTransaction.scala index 2839e569770..428b165d33d 100644 --- a/src/main/scala/scorex/transaction/assets/IssueTransaction.scala +++ b/src/main/scala/scorex/transaction/assets/IssueTransaction.scala @@ -29,6 +29,7 @@ trait IssueTransaction extends ProvenTransaction { override val json = Coeval.evalOnce( jsonBase() ++ Json.obj( "version" -> version, + "assetId" -> assetId().base58, "name" -> new String(name, StandardCharsets.UTF_8), "quantity" -> quantity, "reissuable" -> reissuable, diff --git a/src/main/scala/scorex/transaction/transfer/TransferTransaction.scala b/src/main/scala/scorex/transaction/transfer/TransferTransaction.scala index 71bbaaeb3fd..c87ed924614 100644 --- a/src/main/scala/scorex/transaction/transfer/TransferTransaction.scala +++ b/src/main/scala/scorex/transaction/transfer/TransferTransaction.scala @@ -29,6 +29,7 @@ trait TransferTransaction extends ProvenTransaction { "recipient" -> recipient.stringRepr, "assetId" -> assetId.map(_.base58), "feeAssetId" -> feeAssetId.map(_.base58), + "feeAsset" -> feeAssetId.map(_.base58), // legacy v0.11.1 compat "amount" -> amount, "attachment" -> Base58.encode(attachment) )) diff --git a/src/test/scala/scorex/api/http/AssetsApiRouteSpec.scala b/src/test/scala/scorex/api/http/AssetsApiRouteSpec.scala index d61d8067fc9..ef472ff5ffe 100644 --- a/src/test/scala/scorex/api/http/AssetsApiRouteSpec.scala +++ b/src/test/scala/scorex/api/http/AssetsApiRouteSpec.scala @@ -92,8 +92,8 @@ class AssetsApiRouteSpec (response \ "decimals").as[Int] shouldBe sillyAssetTx.decimals (response \ "reissuable").as[Boolean] shouldBe sillyAssetTx.reissuable (response \ "quantity").as[BigDecimal] shouldBe sillyAssetDesc.totalVolume - (response \ "script").as[String] shouldBe sillyAssetDesc.script.fold("")(_.bytes().base58) - (response \ "scriptText").as[String] shouldBe "" + (response \ "script").asOpt[String] shouldBe None + (response \ "scriptText").asOpt[String] shouldBe None (response \ "complexity").as[Long] shouldBe 0L (response \ "extraFee").as[Long] shouldBe 0 (response \ "minSponsoredAssetFee").asOpt[Long] shouldBe empty diff --git a/src/test/scala/scorex/transaction/IssueTransactionV1Specification.scala b/src/test/scala/scorex/transaction/IssueTransactionV1Specification.scala index 97b701a9748..b3cf4015d4a 100644 --- a/src/test/scala/scorex/transaction/IssueTransactionV1Specification.scala +++ b/src/test/scala/scorex/transaction/IssueTransactionV1Specification.scala @@ -34,6 +34,7 @@ class IssueTransactionV1Specification extends PropSpec with PropertyChecks with "timestamp": 1526287561757, "version": 1, "signature": "28kE1uN1pX2bwhzr9UHw5UuB9meTFEDFgeunNgy6nZWpHX4pzkGYotu8DhQ88AdqUG6Yy5wcXgHseKPBUygSgRMJ", + "assetId": "9ekQuYn92natMnMq8KqeGK3Nn7cpKd3BvPEGgD6fFyyz", "name": "Gigacoin", "quantity": 10000000000, "reissuable": true, @@ -58,7 +59,7 @@ class IssueTransactionV1Specification extends PropSpec with PropertyChecks with .right .get - js shouldEqual tx.json() + tx.json() shouldEqual js } } diff --git a/src/test/scala/scorex/transaction/IssueTransactionV2Specification.scala b/src/test/scala/scorex/transaction/IssueTransactionV2Specification.scala index b31892d0264..605264fe927 100644 --- a/src/test/scala/scorex/transaction/IssueTransactionV2Specification.scala +++ b/src/test/scala/scorex/transaction/IssueTransactionV2Specification.scala @@ -39,6 +39,7 @@ class IssueTransactionV2Specification extends PropSpec with PropertyChecks with "43TCfWBa6t2o2ggsD4bU9FpvH3kmDbSBWKE1Z6B5i5Ax5wJaGT2zAvBihSbnSS3AikZLcicVWhUk1bQAMWVzTG5g" ], "version": 2, + "assetId": "2ykNAo5JrvNCcL8PtCmc9pTcNtKUy2PjJkrFdRvTfUf4", "name": "Gigacoin", "quantity": 10000000000, "reissuable": true, @@ -66,7 +67,7 @@ class IssueTransactionV2Specification extends PropSpec with PropertyChecks with .right .get - js shouldEqual tx.json() + tx.json() shouldEqual js } } diff --git a/src/test/scala/scorex/transaction/TransferTransactionV1Specification.scala b/src/test/scala/scorex/transaction/TransferTransactionV1Specification.scala index 5d912cb9a77..262fa60bf1f 100644 --- a/src/test/scala/scorex/transaction/TransferTransactionV1Specification.scala +++ b/src/test/scala/scorex/transaction/TransferTransactionV1Specification.scala @@ -47,6 +47,7 @@ class TransferTransactionV1Specification extends PropSpec with PropertyChecks wi "recipient": "3My3KZgFQ3CrVHgz6vGRt8687sH4oAA1qp8", "assetId": null, "feeAssetId": null, + "feeAsset": null, "amount": 1900000, "attachment": "4t2Xazb2SX" } @@ -67,6 +68,6 @@ class TransferTransactionV1Specification extends PropSpec with PropertyChecks wi .right .get - js shouldEqual tx.json() + tx.json() shouldEqual js } } diff --git a/src/test/scala/scorex/transaction/TransferTransactionV2Specification.scala b/src/test/scala/scorex/transaction/TransferTransactionV2Specification.scala index d8e85e4f49f..954a7bd8f49 100644 --- a/src/test/scala/scorex/transaction/TransferTransactionV2Specification.scala +++ b/src/test/scala/scorex/transaction/TransferTransactionV2Specification.scala @@ -65,6 +65,7 @@ class TransferTransactionV2Specification extends PropSpec with PropertyChecks wi "recipient": "3My3KZgFQ3CrVHgz6vGRt8687sH4oAA1qp8", "assetId": null, "feeAssetId": null, + "feeAsset": null, "amount": 100000000, "attachment": "4t2Xazb2SX"} """) @@ -85,6 +86,6 @@ class TransferTransactionV2Specification extends PropSpec with PropertyChecks wi .right .get - js shouldEqual tx.json() + tx.json() shouldEqual js } } From 4cdd39370cf7c3f86b609abae89bf40bc367aa3f Mon Sep 17 00:00:00 2001 From: Alexandr M Date: Fri, 25 May 2018 16:12:01 +0300 Subject: [PATCH 50/52] NODE-129: Round block delay in NxtPoSCalc --- .../com/wavesplatform/consensus/PoSCalculator.scala | 2 +- .../scala/com/wavesplatform/consensus/PoSSelector.scala | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala index 4b34753a74a..ef9a261015d 100644 --- a/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala +++ b/src/main/scala/com/wavesplatform/consensus/PoSCalculator.scala @@ -70,7 +70,7 @@ object NxtPoSCalculator extends PoSCalculator { } } - def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long = ((hit * 1000) / (BigInt(bt) * balance)).toLong + def calculateDelay(hit: BigInt, bt: Long, balance: Long): Long = Math.ceil((BigDecimal(hit) / (BigDecimal(bt) * balance)).toDouble).toLong * 1000 } diff --git a/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala b/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala index 9c2b55904d9..009d083c6f1 100644 --- a/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala +++ b/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala @@ -44,17 +44,18 @@ class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { def validateBlockDelay(height: Int, block: Block, parent: Block, effectiveBalance: Long): Either[ValidationError, Unit] = { getValidBlockDelay(height, block.signerData.generator.publicKey, parent.consensusData.baseTarget, effectiveBalance) - .map(_ + parent.timestamp < block.timestamp) - .ensure(GenericError(s"Block time ${block.timestamp} less than expected"))(identity) + .map(_ + parent.timestamp) + .ensureOr(mvt => GenericError(s"Block timestamp ${block.timestamp} less than min valid timestamp $mvt"))(ts => ts <= block.timestamp) .map(_ => ()) } def validateGeneratorSignature(height: Int, block: Block): Either[ValidationError, Unit] = { + val blockGS = block.consensusData.generationSignature.arr blockchain.lastBlock .map(b => generatorSignature(b.consensusData.generationSignature.arr, block.signerData.generator.publicKey)) - .toRight(GenericError("No blocks in blockchain T.T")) - .ensure(GenericError("Generation signatures doesnot match"))(_ sameElements block.consensusData.generationSignature.arr) + .ensureOr(vgs => GenericError(s"Generation signatures doesnot match: Expected = $vgs; Found = $blockGS"))(_ sameElements blockGS) .map(_ => ()) + .toRight(GenericError("No blocks in blockchain")) } def validateBaseTarget(height: Int, block: Block, parent: Block, grandParent: Option[Block]): Either[ValidationError, Unit] = { From 68d12ff12ffe83b382da23082684ce237c91beaf Mon Sep 17 00:00:00 2001 From: Alexey Kiselev Date: Fri, 25 May 2018 16:59:01 +0300 Subject: [PATCH 51/52] NODE-129: Fallback version was set to 0.13.0 and typo was fixed --- build.sbt | 2 +- src/main/scala/com/wavesplatform/consensus/PoSSelector.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 1356d72129f..61a9d9cf5f9 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val versionSource = Def.task { // Please, update the fallback version every major and minor releases. // This version is used then building from sources without Git repository // In case of not updating the version nodes build from headless sources will fail to connect to newer versions - val FallbackVersion = (0, 11, 0) + val FallbackVersion = (0, 13, 0) val versionFile = (sourceManaged in Compile).value / "com" / "wavesplatform" / "Version.scala" val versionExtractor = """(\d+)\.(\d+)\.(\d+).*""".r diff --git a/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala b/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala index 009d083c6f1..620c0b47834 100644 --- a/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala +++ b/src/main/scala/com/wavesplatform/consensus/PoSSelector.scala @@ -53,7 +53,7 @@ class PoSSelector(blockchain: Blockchain, settings: BlockchainSettings) { val blockGS = block.consensusData.generationSignature.arr blockchain.lastBlock .map(b => generatorSignature(b.consensusData.generationSignature.arr, block.signerData.generator.publicKey)) - .ensureOr(vgs => GenericError(s"Generation signatures doesnot match: Expected = $vgs; Found = $blockGS"))(_ sameElements blockGS) + .ensureOr(vgs => GenericError(s"Generation signatures does not match: Expected = $vgs; Found = $blockGS"))(_ sameElements blockGS) .map(_ => ()) .toRight(GenericError("No blocks in blockchain")) } From ae236a67aeefd5f91d2c6a39b1e66bdf08100817 Mon Sep 17 00:00:00 2001 From: Alexandr M Date: Fri, 25 May 2018 17:19:15 +0300 Subject: [PATCH 52/52] NODE-129: Remove offset rounding in miner --- src/main/scala/com/wavesplatform/mining/Miner.scala | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/main/scala/com/wavesplatform/mining/Miner.scala b/src/main/scala/com/wavesplatform/mining/Miner.scala index c08096bb3a6..3d071508e09 100644 --- a/src/main/scala/com/wavesplatform/mining/Miner.scala +++ b/src/main/scala/com/wavesplatform/mining/Miner.scala @@ -300,8 +300,9 @@ class MinerImpl(allChannels: ChannelGroup, _ <- checkAge(height, blockchainUpdater.lastBlockTimestamp.get) _ <- checkScript(account) balanceAndTs <- nextBlockGenerationTime(blockchainSettings.functionalitySettings, height, lastBlock, account) - (balance, ts) = balanceAndTs - offset = calcOffset(timeService, ts, minerSettings.minimalBlockGenerationOffset) + (balance, ts) = balanceAndTs + calculatedOffset = ts - timeService.correctedTime() + offset = Math.max(calculatedOffset, minerSettings.minimalBlockGenerationOffset.toMillis).millis } yield (offset, balance) } match { case Right((offset, balance)) => @@ -368,12 +369,6 @@ object Miner { override val state = MinerDebugInfo.Disabled } - def calcOffset(timeService: Time, calculatedTimestamp: Long, minimalBlockGenerationOffset: FiniteDuration): FiniteDuration = { - val calculatedGenerationTimestamp = (Math.ceil(calculatedTimestamp / 1000.0) * 1000).toLong - val calculatedOffset = calculatedGenerationTimestamp - timeService.correctedTime() - Math.max(minimalBlockGenerationOffset.toMillis, calculatedOffset).millis - } - sealed trait MicroblockMiningResult case object Stop extends MicroblockMiningResult