From 9f56c1301a5ff0829e34a907ed3747685125b926 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Fri, 7 Jan 2022 02:31:17 -0800 Subject: [PATCH 1/3] add userdata&algocustomdata --- src/index.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/index.js b/src/index.js index 3dd47fb..1ce5192 100644 --- a/src/index.js +++ b/src/index.js @@ -73,6 +73,9 @@ async function main({ workflow: workflowPath, path, workflowid, verbose }) { const ddoDir = `${path}/ddos` try { fs.mkdirSync(ddoDir) } catch (e) { console.error(e) } + const algoCustomDataDir = `${path}/algocustomdata` + try { fs.mkdirSync(algoCustomDataDir) } + catch (e) { console.error(e) } const outputsDir = `${path}/outputs` try { fs.mkdirSync(outputsDir) @@ -241,6 +244,11 @@ async function dowloadAsset(what, folder, ddoFolder, useAlgorithmNameInsteadOfIn consumeUrl += `&transferTxId=${txId}` consumeUrl += `&consumerAddress=${checksumAddress}` consumeUrl += `&signature=${signature}` + if (what.remote.userdata) + consumeUrl += '&userdata=' + encodeURI(JSON.stringify(what.remote.userdata)) + if (what.remote.algouserdata) + consumeUrl += '&userdata=' + encodeURI(JSON.stringify(what.remote.algouserdata)) + if (i == 0 && useAlgorithmNameInsteadOfIndex) filePath = folder + 'algorithm' else filePath = folder + i const downloadresult = await downloadurl(consumeUrl, filePath) @@ -257,5 +265,10 @@ async function dowloadAsset(what, folder, ddoFolder, useAlgorithmNameInsteadOfIn else { console.log("No url or remote key? Skipping this input ") } + // write algo custom data if exists + if('algocustomdata' in what){ + fs.writeFileSync(algoCustomDataDir + '/' + what.id.replace('did:op:', ''), JSON.stringify(what.algocustomdata)); + console.log("AlgoCustomData saved to " + algoCustomDataDir + '/' + what.id) + } return (true) } From e08e062391571cb8a8b4490581ee176298f9915b Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Fri, 7 Jan 2022 03:10:07 -0800 Subject: [PATCH 2/3] fix algocustomdata & timeouts --- src/index.js | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/index.js b/src/index.js index 1ce5192..590316d 100644 --- a/src/index.js +++ b/src/index.js @@ -73,9 +73,6 @@ async function main({ workflow: workflowPath, path, workflowid, verbose }) { const ddoDir = `${path}/ddos` try { fs.mkdirSync(ddoDir) } catch (e) { console.error(e) } - const algoCustomDataDir = `${path}/algocustomdata` - try { fs.mkdirSync(algoCustomDataDir) } - catch (e) { console.error(e) } const outputsDir = `${path}/outputs` try { fs.mkdirSync(outputsDir) @@ -111,6 +108,11 @@ async function main({ workflow: workflowPath, path, workflowid, verbose }) { // no need to download algo if input failed const algos = stages.reduce((acc, { algorithm }) => [...acc, algorithm], []) const algoPath = transformationsDir + '/' + // write algo custom data if exists + if('algocustomdata' in algos[0]){ + fs.writeFileSync(inputsDir + '/algoCustomData.json', JSON.stringify(algos[0].algocustomdata)); + console.log("AlgoCustomData saved to " + inputsDir + '/algoCustomData.json') + } if (algos[0].rawcode != null) { if (algos[0].rawcode.length > 10) { fs.writeFileSync(algoPath + 'algorithm', algos[0].rawcode) @@ -166,7 +168,11 @@ async function main({ workflow: workflowPath, path, workflowid, verbose }) { async function downloadurl(url, target) { console.log('Downloading ' + url + ' to ' + target) try { - await pipeline(got.stream(url), fs.createWriteStream(target)) + await pipeline(got.stream(url, { + timeout: { + request: 10000 + } + }), fs.createWriteStream(target)) return true } catch (e) { console.log('Download error:') @@ -227,15 +233,25 @@ async function dowloadAsset(what, folder, ddoFolder, useAlgorithmNameInsteadOfIn const { attributes } = ddo.findServiceByType('metadata') const service = ddo.findServiceById(serviceIndex) const { files } = attributes.main + console.log("Setting provider to: " + service.serviceEndpoint) await ocean.provider.setBaseUrl(service.serviceEndpoint) + let urlPath + try{ + urlPath = ocean.provider.getDownloadEndpoint().urlPath + } + catch(e){ + console.error("Failed to get provider download endpoint") + console.error(e) + return false + } for (let i = 0; i < files.length; i++) { await ocean.provider.getNonce(account) const hash = Web3.utils.utf8ToHex(what.id + ocean.provider.nonce) const sign = web3Accounts.sign(hash, process.env.PRIVATE_KEY) const checksumAddress = Web3.utils.toChecksumAddress(account) const signature = sign.signature - let consumeUrl = ocean.provider.getDownloadEndpoint().urlPath + let consumeUrl = urlPath consumeUrl += `?fileIndex=${files[i].index}` consumeUrl += `&documentId=${what.id}` consumeUrl += `&serviceId=${serviceIndex}` @@ -265,10 +281,5 @@ async function dowloadAsset(what, folder, ddoFolder, useAlgorithmNameInsteadOfIn else { console.log("No url or remote key? Skipping this input ") } - // write algo custom data if exists - if('algocustomdata' in what){ - fs.writeFileSync(algoCustomDataDir + '/' + what.id.replace('did:op:', ''), JSON.stringify(what.algocustomdata)); - console.log("AlgoCustomData saved to " + algoCustomDataDir + '/' + what.id) - } return (true) } From 3dfbb588112185843a3af8c37b43c18756da5fb4 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Fri, 7 Jan 2022 03:11:16 -0800 Subject: [PATCH 3/3] add log --- src/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.js b/src/index.js index 590316d..9425fd0 100644 --- a/src/index.js +++ b/src/index.js @@ -173,6 +173,7 @@ async function downloadurl(url, target) { request: 10000 } }), fs.createWriteStream(target)) + console.log("Downloaded OK") return true } catch (e) { console.log('Download error:')