[#1475] Adopt transaction data requests

- RustBackend extended for the new 2 rust methods 'transactionDataRequests' and 'setTransactionStatus' - lightwalletservice extended to add a new method 'getTaddressTxids' - Enhance action has been refactored to handle transactionDataRequests [#1475] Adopt transaction data requests - fixes [#1475] Adopt transaction data requests - Error codes for specific rust and service errors defined - Fix for the txId [#1475] Adopt transaction data requests - Checkpoints added - Code cleanup [#1475] Adopt transaction data requests - bugfixes in the ffi [#1475] Adopt transaction data requests - FFI with fixes [#1475] Adopt transaction data requests - Another FFI update with fixes, this time the final [#1475] Adopt transaction data requests - Fix for the not recognized state of the transaction for FetchTransaction(txId:) [#1475] Adopt transaction data requests - Code cleaned up and polished [#1475] Adopt transaction data requests - Changelog updated [#1475] Adopt transaction data requests - DemoApp settings reverted
Electric-Coin-Company · Aug 18, 2024 · 35c31a4 · 35c31a4
1 parent a3e15f0
commit 35c31a4
Show file tree

Hide file tree

Showing 35 changed files with 449 additions and 39 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,21 @@ and this library adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `Synchronizer.exchangeRateUSDStream`. Prices are queried over Tor (to hide the wallet's
   IP address).
 
+## Changed
+
+### [#1475] Adopt transaction data requests
+- The transaction history is now processed using `transaction data requests`, which are fetched every 1,000 blocks during longer syncs or with each sync loop when a new block is mined.
+
+## Checkpoints
+
+Mainnet
+
+````
+Sources/ZcashLightClientKit/Resources/checkpoints/mainnet/2562500.json
+...
+Sources/ZcashLightClientKit/Resources/checkpoints/mainnet/2610000.json
+````
+
 # 2.1.12 - 2024-07-04
 
 ## Fixed

diff --git a/...cashLightClientSample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/...cashLightClientSample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -176,7 +176,7 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/zcash-hackworks/zcash-light-client-ffi",
       "state" : {
-        "revision" : "4de1b42f99aebfc5e4f0340da8a66a4f719db9a6"
+        "revision" : "3ccafcddfe51918239d157fd839476959413840f"
       }
     }
   ],

diff --git a/Package.swift b/Package.swift
@@ -18,7 +18,8 @@ let package = Package(
         .package(url: "https://github.com/stephencelis/SQLite.swift.git", from: "0.14.1"),
         // .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", exact: "0.8.1")
         // Compiled from 2516a94f8bdc540d951c38b66e9c07e2b8c29cb4
-        .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", revision: "4de1b42f99aebfc5e4f0340da8a66a4f719db9a6")
+//        .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", branch: "ffi_transaction_requests_preview")
+        .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", revision: "3ccafcddfe51918239d157fd839476959413840f")
     ],
     targets: [
         .target(

diff --git a/Sources/ZcashLightClientKit/Block/Enhance/BlockEnhancer.swift b/Sources/ZcashLightClientKit/Block/Enhance/BlockEnhancer.swift
@@ -60,21 +60,24 @@ struct BlockEnhancerImpl {
     let rustBackend: ZcashRustBackendWelding
     let transactionRepository: TransactionRepository
     let metrics: SDKMetrics
+    let service: LightWalletService
     let logger: Logger
 
-    private func enhance(transaction: ZcashTransaction.Overview) async throws -> ZcashTransaction.Overview {
-        logger.debug("Zoom.... Enhance... Tx: \(transaction.rawID.toHexStringTxId())")
+    private func enhance(txId: Data) async throws -> ZcashTransaction.Overview {
+        logger.debug("Zoom.... Enhance... Tx: \(txId.toHexStringTxId())")
 
-        let fetchedTransaction = try await blockDownloaderService.fetchTransaction(txId: transaction.rawID)
+        let fetchedTransaction = try await blockDownloaderService.fetchTransaction(txId: txId)
 
-        let transactionID = fetchedTransaction.rawID.toHexStringTxId()
-        let block = String(describing: transaction.minedHeight)
-        logger.debug("Decrypting and storing transaction id: \(transactionID) block: \(block)")
-
-        try await rustBackend.decryptAndStoreTransaction(
-            txBytes: fetchedTransaction.raw.bytes,
-            minedHeight: Int32(fetchedTransaction.minedHeight)
-        )
+        if fetchedTransaction.minedHeight == -1 {
+            try await rustBackend.setTransactionStatus(txId: fetchedTransaction.rawID, status: .txidNotRecognized)
+        } else if fetchedTransaction.minedHeight == 0 {
+            try await rustBackend.setTransactionStatus(txId: fetchedTransaction.rawID, status: .notInMainChain)
+        } else if fetchedTransaction.minedHeight > 0 {
+            try await rustBackend.decryptAndStoreTransaction(
+                txBytes: fetchedTransaction.raw.bytes,
+                minedHeight: Int32(fetchedTransaction.minedHeight)
+            )
+        }
 
         return try await transactionRepository.find(rawID: fetchedTransaction.rawID)
     }
@@ -92,42 +95,56 @@ extension BlockEnhancerImpl: BlockEnhancer {
         // fetch transactions
         do {
             let startTime = Date()
-            let transactions = try await transactionRepository.find(in: range, limit: Int.max, kind: .all)
+            let transactionDataRequests = try await rustBackend.transactionDataRequests()
 
-            guard !transactions.isEmpty else {
-                logger.debug("no transactions detected on range: \(range.lowerBound)...\(range.upperBound)")
-                logger.sync("No transactions detected on range: \(range.lowerBound)...\(range.upperBound)")
+            guard !transactionDataRequests.isEmpty else {
+                logger.debug("No transaction data requests detected.")
+                logger.sync("No transaction data requests detected.")
                 return nil
             }
 
             let chainTipHeight = try await blockDownloaderService.latestBlockHeight()
-
             let newlyMinedLowerBound = chainTipHeight - ZcashSDK.expiryOffset
-
             let newlyMinedRange = newlyMinedLowerBound...chainTipHeight
 
-            for index in 0 ..< transactions.count {
-                let transaction = transactions[index]
+            for index in 0 ..< transactionDataRequests.count {
+                let transactionDataRequest = transactionDataRequests[index]
                 var retry = true
-                
+
                 while retry && retries < maxRetries {
                     try Task.checkCancellation()
                     do {
-                        let confirmedTx = try await enhance(transaction: transaction)
-                        retry = false
-
-                        let progress = EnhancementProgress(
-                            totalTransactions: transactions.count,
-                            enhancedTransactions: index + 1,
-                            lastFoundTransaction: confirmedTx,
-                            range: range,
-                            newlyMined: confirmedTx.isSentTransaction && newlyMinedRange.contains(confirmedTx.minedHeight ?? 0)
-                        )
-
-                        await didEnhance(progress)
+                        switch transactionDataRequest {
+                        case .getStatus(let txId), .enhancement(let txId):
+                            let confirmedTx = try await enhance(txId: txId.data)
+                            retry = false
+
+                            let progress = EnhancementProgress(
+                                totalTransactions: transactionDataRequests.count,
+                                enhancedTransactions: index + 1,
+                                lastFoundTransaction: confirmedTx,
+                                range: range,
+                                newlyMined: confirmedTx.isSentTransaction && newlyMinedRange.contains(confirmedTx.minedHeight ?? 0)
+                            )
+
+                            await didEnhance(progress)
+                        case .spendsFromAddress(let sfa):
+                            var filter = TransparentAddressBlockFilter()
+                            filter.address = sfa.address
+                            filter.range = BlockRange(startHeight: Int(sfa.blockRangeStart), endHeight: Int(sfa.blockRangeEnd))
+                            let stream = service.getTaddressTxids(filter)
+
+                            for try await rawTransaction in stream {
+                                try await rustBackend.decryptAndStoreTransaction(
+                                    txBytes: rawTransaction.data.bytes,
+                                    minedHeight: Int32(rawTransaction.height)
+                                )
+                            }
+                            retry = false
+                        }
                     } catch {
                         retries += 1
-                        logger.error("could not enhance txId \(transaction.rawID.toHexStringTxId()) - Error: \(error)")
+                        logger.error("could not enhance transactionDataRequest \(transactionDataRequest) - Error: \(error)")
                         if retries > maxRetries {
                             throw error
                         }
@@ -137,7 +154,7 @@ extension BlockEnhancerImpl: BlockEnhancer {
 
             let endTime = Date()
             let diff = endTime.timeIntervalSince1970 - startTime.timeIntervalSince1970
-            let logMsg = "Enhanced \(transactions.count) transaction(s) in \(diff) for range \(range.lowerBound)...\(range.upperBound)"
+            let logMsg = "Enhanced \(transactionDataRequests.count) transaction data requests in \(diff)"
             logger.sync(logMsg)
             metrics.actionDetail(logMsg, for: .enhance)
         } catch {

diff --git a/Sources/ZcashLightClientKit/Error/ZcashError.swift b/Sources/ZcashLightClientKit/Error/ZcashError.swift
@@ -61,6 +61,9 @@ public enum ZcashError: Equatable, Error {
     /// LightWalletService.getSubtreeRoots failed.
     /// ZSRVC0009
     case serviceSubtreeRootsStreamFailed(_ error: LightWalletServiceError)
+    /// LightWalletService.getTaddressTxids failed.
+    /// ZSRVC0010
+    case serviceGetTaddressTxidsFailed(_ error: LightWalletServiceError)
     /// SimpleConnectionProvider init of Connection failed.
     /// ZSCPC0001
     case simpleConnectionProvider(_ error: Error)
@@ -352,6 +355,11 @@ public enum ZcashError: Equatable, Error {
     /// sourcery: code="ZRUST0063"
     /// ZRUST0063
     case rustTorClientGet(_ rustError: String)
+    /// Error from rust layer when calling ZcashRustBackend.transactionDataRequests
+    /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0064"
+    /// ZRUST0064
+    case rustTransactionDataRequests(_ rustError: String)
     /// SQLite query failed when fetching all accounts from the database.
     /// - `sqliteError` is error produced by SQLite library.
     /// ZADAO0001
@@ -646,6 +654,7 @@ public enum ZcashError: Equatable, Error {
         case .serviceFetchUTXOsFailed: return "LightWalletService.fetchUTXOs failed."
         case .serviceBlockStreamFailed: return "LightWalletService.blockStream failed."
         case .serviceSubtreeRootsStreamFailed: return "LightWalletService.getSubtreeRoots failed."
+        case .serviceGetTaddressTxidsFailed: return "LightWalletService.getTaddressTxids failed."
         case .simpleConnectionProvider: return "SimpleConnectionProvider init of Connection failed."
         case .saplingParamsInvalidSpendParams: return "Downloaded file with sapling spending parameters isn't valid."
         case .saplingParamsInvalidOutputParams: return "Downloaded file with sapling output parameters isn't valid."
@@ -720,6 +729,7 @@ public enum ZcashError: Equatable, Error {
         case .rustPutOrchardSubtreeRoots: return "Error from rust layer when calling ZcashRustBackend.putOrchardSubtreeRoots"
         case .rustTorClientInit: return "Error from rust layer when calling TorClient.init"
         case .rustTorClientGet: return "Error from rust layer when calling TorClient.get"
+        case .rustTransactionDataRequests: return "Error from rust layer when calling ZcashRustBackend.transactionDataRequests"
         case .accountDAOGetAll: return "SQLite query failed when fetching all accounts from the database."
         case .accountDAOGetAllCantDecode: return "Fetched accounts from SQLite but can't decode them."
         case .accountDAOFindBy: return "SQLite query failed when seaching for accounts in the database."
@@ -829,6 +839,7 @@ public enum ZcashError: Equatable, Error {
         case .serviceFetchUTXOsFailed: return .serviceFetchUTXOsFailed
         case .serviceBlockStreamFailed: return .serviceBlockStreamFailed
         case .serviceSubtreeRootsStreamFailed: return .serviceSubtreeRootsStreamFailed
+        case .serviceGetTaddressTxidsFailed: return .serviceGetTaddressTxidsFailed
         case .simpleConnectionProvider: return .simpleConnectionProvider
         case .saplingParamsInvalidSpendParams: return .saplingParamsInvalidSpendParams
         case .saplingParamsInvalidOutputParams: return .saplingParamsInvalidOutputParams
@@ -903,6 +914,7 @@ public enum ZcashError: Equatable, Error {
         case .rustPutOrchardSubtreeRoots: return .rustPutOrchardSubtreeRoots
         case .rustTorClientInit: return .rustTorClientInit
         case .rustTorClientGet: return .rustTorClientGet
+        case .rustTransactionDataRequests: return .rustTransactionDataRequests
         case .accountDAOGetAll: return .accountDAOGetAll
         case .accountDAOGetAllCantDecode: return .accountDAOGetAllCantDecode
         case .accountDAOFindBy: return .accountDAOFindBy

diff --git a/Sources/ZcashLightClientKit/Error/ZcashErrorCode.swift b/Sources/ZcashLightClientKit/Error/ZcashErrorCode.swift
@@ -41,6 +41,8 @@ public enum ZcashErrorCode: String {
     case serviceBlockStreamFailed = "ZSRVC0000"
     /// LightWalletService.getSubtreeRoots failed.
     case serviceSubtreeRootsStreamFailed = "ZSRVC0009"
+    /// LightWalletService.getTaddressTxids failed.
+    case serviceGetTaddressTxidsFailed = "ZSRVC0010"
     /// SimpleConnectionProvider init of Connection failed.
     case simpleConnectionProvider = "ZSCPC0001"
     /// Downloaded file with sapling spending parameters isn't valid.
@@ -189,6 +191,8 @@ public enum ZcashErrorCode: String {
     case rustTorClientInit = "ZRUST0062"
     /// Error from rust layer when calling TorClient.get
     case rustTorClientGet = "ZRUST0063"
+    /// Error from rust layer when calling ZcashRustBackend.transactionDataRequests
+    case rustTransactionDataRequests = "ZRUST0064"
     /// SQLite query failed when fetching all accounts from the database.
     case accountDAOGetAll = "ZADAO0001"
     /// Fetched accounts from SQLite but can't decode them.

diff --git a/Sources/ZcashLightClientKit/Error/ZcashErrorCodeDefinition.swift b/Sources/ZcashLightClientKit/Error/ZcashErrorCodeDefinition.swift
@@ -80,6 +80,9 @@ enum ZcashErrorDefinition {
     /// LightWalletService.getSubtreeRoots failed.
     // sourcery: code="ZSRVC0009"
     case serviceSubtreeRootsStreamFailed(_ error: LightWalletServiceError)
+    /// LightWalletService.getTaddressTxids failed.
+    // sourcery: code="ZSRVC0010"
+    case serviceGetTaddressTxidsFailed(_ error: LightWalletServiceError)
 
     // MARK: SQLite connection
 
@@ -375,6 +378,10 @@ enum ZcashErrorDefinition {
     /// - `rustError` contains error generated by the rust layer.
     /// sourcery: code="ZRUST0063"
     case rustTorClientGet(_ rustError: String)
+    /// Error from rust layer when calling ZcashRustBackend.transactionDataRequests
+    /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0064"
+    case rustTransactionDataRequests(_ rustError: String)
 
     // MARK: - Account DAO
 

diff --git a/Sources/ZcashLightClientKit/Model/Proposal.swift b/Sources/ZcashLightClientKit/Model/Proposal.swift
@@ -35,7 +35,7 @@ public extension Proposal {
     /// passed to `Synchronizer.createProposedTransactions`. It should never be called in
     /// production code.
     static func testOnlyFakeProposal(totalFee: UInt64) -> Self {
-        var ffiProposal = FfiProposal()
+        let ffiProposal = FfiProposal()
         var balance = FfiTransactionBalance()
 
         balance.feeRequired = totalFee

diff --git a/Sources/ZcashLightClientKit/Model/TransactionDataRequest.swift b/Sources/ZcashLightClientKit/Model/TransactionDataRequest.swift
@@ -0,0 +1,26 @@
+//
+//  TransactionDataRequest.swift
+//
+//
+//  Created by Lukáš Korba on 08-15-2024.
+//
+
+import Foundation
+
+struct SpendsFromAddress: Equatable {
+    let address: String
+    let blockRangeStart: UInt32
+    let blockRangeEnd: Int64
+}
+
+enum TransactionDataRequest: Equatable {
+    case getStatus([UInt8])
+    case enhancement([UInt8])
+    case spendsFromAddress(SpendsFromAddress)
+}
+
+enum TransactionStatus: Equatable {
+    case txidNotRecognized
+    case notInMainChain
+    case mined(BlockHeight)
+}
diff --git a/Sources/ZcashLightClientKit/Modules/Service/GRPC/LightWalletGRPCService.swift b/Sources/ZcashLightClientKit/Modules/Service/GRPC/LightWalletGRPCService.swift
@@ -193,7 +193,11 @@ extension LightWalletGRPCService: LightWalletService {
 
         do {
             let rawTx = try await compactTxStreamer.getTransaction(txFilter)
-            return ZcashTransaction.Fetched(rawID: txId, minedHeight: BlockHeight(rawTx.height), raw: rawTx.data)
+            return ZcashTransaction.Fetched(
+                rawID: txId,
+                minedHeight: rawTx.height == UInt64.max ? BlockHeight(-1) : BlockHeight(rawTx.height),
+                raw: rawTx.data
+            )
         } catch {
             let serviceError = error.mapToServiceError()
             throw ZcashError.serviceFetchTransactionFailed(serviceError)
@@ -280,6 +284,21 @@ extension LightWalletGRPCService: LightWalletService {
         try await compactTxStreamer.getTreeState(id)
     }
 
+    func getTaddressTxids(_ request: TransparentAddressBlockFilter) -> AsyncThrowingStream<RawTransaction, Error> {
+        let stream = compactTxStreamer.getTaddressTxids(request)
+        var iterator = stream.makeAsyncIterator()
+
+        return AsyncThrowingStream() {
+            do {
+                guard let rawTransaction = try await iterator.next() else { return nil }
+                return rawTransaction
+            } catch {
+                let serviceError = error.mapToServiceError()
+                throw ZcashError.serviceGetTaddressTxidsFailed(serviceError)
+            }
+        }
+    }
+
     func closeConnection() {
         _ = channel.close()
     }

diff --git a/Sources/ZcashLightClientKit/Modules/Service/LightWalletService.swift b/Sources/ZcashLightClientKit/Modules/Service/LightWalletService.swift
@@ -198,4 +198,6 @@ protocol LightWalletService: AnyObject {
     func getSubtreeRoots(_ request: GetSubtreeRootsArg) -> AsyncThrowingStream<SubtreeRoot, Error>
 
     func getTreeState(_ id: BlockID) async throws -> TreeState
+
+    func getTaddressTxids(_ request: TransparentAddressBlockFilter) -> AsyncThrowingStream<RawTransaction, Error>
 }
diff --git a/Sources/ZcashLightClientKit/Resources/checkpoints/mainnet/2562500.json b/Sources/ZcashLightClientKit/Resources/checkpoints/mainnet/2562500.json
@@ -0,0 +1,8 @@
+{
+  "network": "main",
+  "height": "2562500",
+  "hash": "0000000001c0df88bddeaf43751e3372c0ef8bc367d530686cae28997ddf7c01",
+  "time": 1720007743,
+  "saplingTree": "01ecd49d8483d7506316eafd26dccd90c825ccf227a3d85aa369d8503d4020b51c001f01735e5e2ceec49fefd01d6384b550612dc95de14244e6189ae38dc347bb643c620001aa7364947a2c025c3c9a7e3b8b6df69dd3abf49a7ea5fc608f5dd3ae8d13b35101136827bc5bc212f513979f0bfd8f4903a53abcb480ee8c12fba8cd6f93246c6501664e69f8c163e6b6e760f77704402dba169eae51e1cb12c19dd88693c6d48764000001c65e700bee3337938031ed5fd369f58300db4824ceb0a581db032f56d769d06f014cc5db5bb4a981cb133330b004496bc06459b07ac147fefe85647ea62048a05700011b0a8a46c6af73471be28cb8883c3e598de138726017dbac884e66badf7aff5a019e38bfb9d2e5f571397fc8a1f97ae6307e9f908740f806d05d08f8e089ec663c000001cdfe2a600d8672055d7aa8fb49484326362e8cf91225c95874110ce4852331170139f4db89ea9c70bec114669298893398d5cbb526af3e27fe826f7f5a9294674e017f744067a47ada432e380f8bd4835c1e7c461dde6d4569a4379673491e33540a01d1b36bbba8e6e1be8f09baf2b829bafc4ccd89ad25fb730d2b8a995b60fc3a6701d8ccea507421a590ed38116b834189cdc22421b4764179fa4e364803dfa66d56018cf6f5034a55fed59d1d832620916a43c756d2379e50ef9440fc4c3e7a29aa2300011619f99023a69bb647eab2d2aa1a73c3673c74bb033c3c4930eacda19e6fd93b0000000160272b134ca494b602137d89e528c751c06d3ef4a87a45f33af343c15060cc1e0000000000",
+  "orchardTree": "011c142041aab9c2c26dae0840ef30fddf54e98bc0dbe9b6b7365e93377094230101d458c93a33379d22639c8eea7ea0559d77f19012f2b5622b72537ceea8dec2171f017e8174987aec32ff6b4d8020868384cf50258cc583a8f79c55e2368aa09bf6100001bbcfe25a7ddde2ba4aab8ea76742e627642e57f1c38fa22969d90dc28ef28622000001476fde36e8173fc6c1e060a0eadbb886b85355ed3c6cbd96bc4d2b6b36d37038000135c64a501430280fe1229092489b00fd19e3c5ca50db1d439cdf85db61cdd009018e495734e0a3c74b5d428c7e30d7f4b53ca39d9dc21f5c30159264927c27470d0001c435d392d35c9387e1c95726d8c43265d9bdbc87f28be696a7c7436b28bef40d0001c6863eedb51a0fecacb4b34876e2baa848121debfb8182e28073319eb8c8b93c014a25173fb726fea7e30f7be206b2154e91599fb43dcb2d99584cd897a7283f3f014085ca103411cf5d663ab76f084f16ff2b1080a5362c23b613e863b2c0749014012c8e24249acb30263d489e042527a751ab20eefdd6f28d6345d904d76e419e0300000160040850b766b126a2b4843fcdfdffa5d5cab3f53bc860a3bef68958b5f066170001cc2dcaa338b312112db04b435a706d63244dd435238f0aa1e9e1598d35470810012dcc4273c8a0ed2337ecf7879380a07e7d427c7f9d82e538002bd1442978402c01daf63debf5b40df902dae98dadc029f281474d190cddecef1b10653248a234150001e2bca6a8d987d668defba89dc082196a922634ed88e065c669e526bb8815ee1b000000000000"
+}