Merge pull request #982 from golemfactory/mgordel/JST-984/exe-unit-re…

…name Rename WorkContext to ExeUnit
golemfactory · Jun 20, 2024 · 0909f3b · 0909f3b
2 parents ec78913 + 042c76b
commit 0909f3b
Show file tree

Hide file tree

Showing 58 changed files with 383 additions and 1,889 deletions.
diff --git a/README.md b/README.md
@@ -31,6 +31,7 @@
     - [Custom filters](#custom-filters)
     - [Custom ranking of proposals](#custom-ranking-of-proposals)
     - [Uploading local images to the provider](#uploading-local-images-to-the-provider)
+    - [Setup and teardown methods](#setup-and-teardown-methods)
   - [Going further](#going-further)
   - [More examples](#more-examples)
   - [Debugging](#debugging)
@@ -356,6 +357,29 @@ const order: MarketOrderSpec = {
 
 [Check the full example](./examples/advanced//local-image/)
 
+### Setup and teardown methods
+
+You can define a setup method that will be executed the first time a provider is rented and a teardown method
+that will be executed before the rental is done. This is useful when you want to avoid doing the same work
+multiple times when running multiple tasks on the same provider.
+
+```ts
+// I want to upload a big file to each provider only once
+const setup: LifecycleFunction = async (exe) => exe.uploadFile("./big-file.txt", "/golem/work/big-file.txt");
+
+// I want to remove the file after I'm done
+const teardown: LifecycleFunction = async (exe) => exe.run("rm /golem/work/big-file.txt");
+
+const pool = await glm.manyOf({
+  order,
+  concurrency,
+  setup,
+  teardown,
+});
+```
+
+[Check the full example](./examples/advanced/setup-and-teardown.ts)
+
 <!--
 TODO:
  ### Market scan

diff --git a/examples/advanced/local-image/serveLocalGvmi.ts b/examples/advanced/local-image/serveLocalGvmi.ts
@@ -39,7 +39,7 @@ const getImagePath = (path: string) => new URL(path, import.meta.url).toString()
       },
     };
 
-    const rental = await glm.oneOf(order);
+    const rental = await glm.oneOf({ order });
     // in our Dockerfile we have created a file called hello.txt, let's read it
     const result = await rental
       .getExeUnit()

diff --git a/examples/advanced/override-module.ts b/examples/advanced/override-module.ts
@@ -58,7 +58,7 @@ const order: MarketOrderSpec = {
 
   try {
     await glm.connect();
-    const rental = await glm.oneOf(order);
+    const rental = await glm.oneOf({ order });
     await rental
       .getExeUnit()
       .then((exe) => exe.run("echo Hello, Golem! 👋"))

diff --git a/examples/advanced/payment-filters.ts b/examples/advanced/payment-filters.ts
@@ -61,7 +61,7 @@ const order: MarketOrderSpec = {
 
   try {
     await glm.connect();
-    const rental = await glm.oneOf(order);
+    const rental = await glm.oneOf({ order });
     await rental
       .getExeUnit()
       .then((exe) => exe.run("echo Hello, Golem! 👋"))

diff --git a/examples/advanced/proposal-filter.ts b/examples/advanced/proposal-filter.ts
@@ -35,7 +35,7 @@ const order: MarketOrderSpec = {
 
   try {
     await glm.connect();
-    const rental = await glm.oneOf(order);
+    const rental = await glm.oneOf({ order });
     await rental
       .getExeUnit()
       .then((exe) => exe.run(`echo [provider:${exe.provider.name}] Hello, Golem! 👋`))

diff --git a/examples/advanced/proposal-predefined-filter.ts b/examples/advanced/proposal-predefined-filter.ts
@@ -33,7 +33,7 @@ const order: MarketOrderSpec = {
 
   try {
     await glm.connect();
-    const rental = await glm.oneOf(order);
+    const rental = await glm.oneOf({ order });
     await rental
       .getExeUnit()
       .then((exe) => exe.run(`echo [provider:${exe.provider.name}] Hello, Golem! 👋`))

diff --git a/examples/advanced/proposal-selector.ts b/examples/advanced/proposal-selector.ts
@@ -41,7 +41,7 @@ const order: MarketOrderSpec = {
 
   try {
     await glm.connect();
-    const rental = await glm.oneOf(order);
+    const rental = await glm.oneOf({ order });
     await rental
       .getExeUnit()
       .then((exe) => exe.run(`echo [provider:${exe.provider.name}] Hello, Golem! 👋`))

diff --git a/examples/advanced/reuse-allocation.ts b/examples/advanced/reuse-allocation.ts
@@ -52,8 +52,8 @@ import { pinoPrettyLogger } from "@golem-sdk/pino-logger";
       },
     };
 
-    const rental1 = await glm.oneOf(firstOrder);
-    const rental2 = await glm.oneOf(secondOrder);
+    const rental1 = await glm.oneOf({ order: firstOrder });
+    const rental2 = await glm.oneOf({ order: secondOrder });
 
     await rental1
       .getExeUnit()

diff --git a/examples/advanced/setup-and-teardown.ts b/examples/advanced/setup-and-teardown.ts
@@ -0,0 +1,101 @@
+/**
+ * This advanced example shows how to use the setup and teardown lifecycle functions
+ * to avoid doing the same work multiple times when running multiple tasks on the same provider.
+ */
+import { MarketOrderSpec, GolemNetwork, LifecycleFunction } from "@golem-sdk/golem-js";
+import { pinoPrettyLogger } from "@golem-sdk/pino-logger";
+
+const order: MarketOrderSpec = {
+  demand: {
+    workload: {
+      imageTag: "golem/examples-openssl:latest",
+    },
+  },
+  market: {
+    rentHours: 0.5,
+    pricing: {
+      model: "linear",
+      maxStartPrice: 0.5,
+      maxCpuPerHourPrice: 1.0,
+      maxEnvPerHourPrice: 0.5,
+    },
+  },
+};
+
+/**
+ * Let's decrypt these messages in parallel on multiple providers.
+ * We know that the messages were encrypted using the aes-256-cbc algorithm
+ * with 5000000 iterations and the password "golemnetwork".
+ */
+const secretMessages = `
+U2FsdGVkX18S4g4iyUzjfqIwP/cGppD86mrVQ0kYerfSe3OnGORuU4nYLG6WXy5KG6vLVLq1N/jnyrwkT8yEBA==
+U2FsdGVkX1+jEgVgmS+xu37tT5OHieX8cioZHPyjUTh+YodWf0In3DaqtFcEfw2cLHIBd94s4nEmONHCs9x4Rg==
+U2FsdGVkX1+JU+fBsnKEGZHGoQpEY/DqlnbCQVg+KgLtkFbtjuHpQbMjnb7iBuj4o4yIYU00jM67+gqn89hrNA==
+U2FsdGVkX1897oplQ7utV9zpx/86GABjUP29Xr/GsahKQ9eRv9GgnzW9BCqHKiFjiB2q2F6gCJINspbuiFY+Fg==
+U2FsdGVkX183p8EUPOZj/QZFQSeIeYNSSfHcRrBF0NXSJ4RfibvT5HtJJ/5I9fVpc1XpbLwDsCW2yFdQKSzbXA==
+U2FsdGVkX184PQiKxx8Sfvl+BOy9JYrBQqdMxWEDc3GBDkEb3qYCWL7FPxZpCEwoZ10MpvY1EKb4lMMxWth6bw==
+U2FsdGVkX1/ujngC/IwK8UAvj41t/FbSHVFiiXI7+KeHoKW3HKcwZYb0E+nncpPC6ZT0DgWLzvDaUyBqOS+tkA==
+U2FsdGVkX19GNHf4ORUAy2PC3MMnvjx7aZSRNSqkW20fZ03Dc2OnZEWBDPa1J4yx
+U2FsdGVkX18iL21PNCohSdFuIOufknLXmINnYf3q15Fl+1vFcRnmC8b9zcrob5Iz/9dNkvrgAeNFmAwWK0bwPw==
+U2FsdGVkX18jQbGQ7KTAaRask5efrXEWvvGhe4jQ0MT9mwwH4ULjvoWDm1mNlsjYtb1nRt0O6iBd4O9moHLbbg==
+`.trim();
+
+(async () => {
+  const glm = new GolemNetwork({
+    logger: pinoPrettyLogger({
+      // for this example we disable info logs to better show the output
+      level: "warn",
+    }),
+  });
+
+  try {
+    await glm.connect();
+
+    // I want to upload my encrypted messages to the provider before I start working
+    const setup: LifecycleFunction = async (exe) =>
+      exe
+        .run(`echo "${secretMessages}" > /golem/work/encrypted-messages.txt`)
+        .then(() => console.log("Uploaded the encrypted messages to the provider %s", exe.provider.name));
+
+    // I want to remove the encrypted messages from the provider after I finish working
+    const teardown: LifecycleFunction = async (exe) =>
+      exe
+        .run("rm /golem/work/encrypted-messages.txt")
+        .then(() => console.log("Removed the encrypted messages from the provider %s", exe.provider.name));
+
+    const pool = await glm.manyOf({
+      concurrency: { max: 3 }, // I want to decrypt in parallel on a maximum of 3 machines simultaneously
+      order,
+      setup,
+      teardown,
+    });
+
+    // map each message to a decryption task
+    const decryptionTasks = new Array(10).fill(null).map((_, i) =>
+      pool.withRental(async (rental) => {
+        const exe = await rental.getExeUnit();
+        const result = await exe.run(
+          `sed '${i + 1}!d' /golem/work/encrypted-messages.txt | \
+              openssl \
+                enc -aes-256-cbc -d -a \
+                -pass pass:golemnetwork \
+                -iter 5000000`,
+        );
+        console.log("Finished decrypting message #%s on the provider %s", i + 1, exe.provider.name);
+        return String(result.stdout).trim();
+      }),
+    );
+    const decryptedMessages = await Promise.all(decryptionTasks);
+    await pool.drainAndClear();
+
+    console.log("Decrypted messages:");
+    for (const message of decryptedMessages) {
+      // display the decrypted messages in light-blue color
+      console.log("\x1b[36m%s\x1b[0m", message);
+    }
+  } catch (err) {
+    console.error("Failed to run the example", err);
+  } finally {
+    await glm.disconnect();
+  }
+})().catch(console.error);
diff --git a/examples/advanced/step-by-step.ts b/examples/advanced/step-by-step.ts
@@ -136,11 +136,11 @@ import { filter, map, switchMap, take } from "rxjs";
     // Start the computation
     // First lets start the activity - this will deploy our image on the provider's machine
     const activity = await glm.activity.createActivity(agreement);
-    // Then let's create a WorkContext, which is a set of utilities to interact with the
+    // Then let's create a ExeUnit, which is a set of utilities to interact with the
     // providers machine, like running commands, uploading files, etc.
-    const ctx = await glm.activity.createWorkContext(activity);
+    const exe = await glm.activity.createExeUnit(activity);
     // Now we can run a simple command on the provider's machine
-    const result = await ctx.run("echo Hello, Golem 👋!");
+    const result = await exe.run("echo Hello, Golem 👋!");
     console.log("Result of the command ran on the provider's machine:", result.stdout);
 
     // We're done, let's clean up

diff --git a/examples/basic/events.ts b/examples/basic/events.ts
@@ -36,16 +36,18 @@ import { pinoPrettyLogger } from "@golem-sdk/pino-logger";
     });
 
     const rental = await glm.oneOf({
-      demand: {
-        workload: { imageTag: "golem/alpine:latest" },
-      },
-      market: {
-        rentHours: 0.5,
-        pricing: {
-          model: "linear",
-          maxStartPrice: 0.5,
-          maxCpuPerHourPrice: 1.0,
-          maxEnvPerHourPrice: 0.5,
+      order: {
+        demand: {
+          workload: { imageTag: "golem/alpine:latest" },
+        },
+        market: {
+          rentHours: 0.5,
+          pricing: {
+            model: "linear",
+            maxStartPrice: 0.5,
+            maxCpuPerHourPrice: 1.0,
+            maxEnvPerHourPrice: 0.5,
+          },
         },
       },
     });

diff --git a/examples/basic/one-of.ts b/examples/basic/one-of.ts
@@ -25,7 +25,7 @@ const order: MarketOrderSpec = {
 
   try {
     await glm.connect();
-    const rental = await glm.oneOf(order);
+    const rental = await glm.oneOf({ order });
     await rental
       .getExeUnit()
       .then((exe) => exe.run("echo Hello, Golem! 👋"))

diff --git a/examples/basic/run-and-stream.ts b/examples/basic/run-and-stream.ts
@@ -30,7 +30,7 @@ const order: MarketOrderSpec = {
 
   try {
     await glm.connect();
-    const rental = await glm.oneOf(order);
+    const rental = await glm.oneOf({ order });
     const exe = await rental.getExeUnit();
 
     const remoteProcess = await exe.runAndStream(

diff --git a/examples/docs-examples/quickstarts/retrievable-task/task.mjs b/examples/docs-examples/quickstarts/retrievable-task/task.mjs
@@ -10,8 +10,8 @@ const job = golem.createJob({
     imageTag: "golem/alpine:latest",
   },
 });
-job.startWork(async (ctx) => {
-  const response = await ctx.run("echo 'Hello, Golem!'");
+job.startWork(async (exe) => {
+  const response = await exe.run("echo 'Hello, Golem!'");
   return response.stdout;
 });
 

diff --git a/examples/experimental/express/server.ts b/examples/experimental/express/server.ts
@@ -53,9 +53,9 @@ app.post("/tts", async (req, res) => {
     console.log("Job succeeded", job.results);
   });
 
-  job.startWork(async (ctx) => {
+  job.startWork(async (exe) => {
     const fileName = `${Math.random().toString(36).slice(2)}.wav`;
-    await ctx
+    await exe
       .beginBatch()
       .run(`espeak "${req.body}" -w /golem/output/output.wav`)
       .downloadFile("/golem/output/output.wav", `public/${fileName}`)

diff --git a/examples/experimental/job/cancel.ts b/examples/experimental/job/cancel.ts
@@ -41,8 +41,8 @@ async function main() {
     console.log("Job ended event emitted");
   });
 
-  job.startWork(async (ctx) => {
-    return String((await ctx.run("echo -n 'Hello Golem!'")).stdout);
+  job.startWork(async (exe) => {
+    return String((await exe.run("echo -n 'Hello Golem!'")).stdout);
   });
 
   console.log("Canceling job...");

diff --git a/examples/experimental/job/getJobById.ts b/examples/experimental/job/getJobById.ts
@@ -37,8 +37,8 @@ function startJob() {
     console.log("Job ended event emitted");
   });
 
-  job.startWork(async (ctx) => {
-    return String((await ctx.run("echo -n 'Hello Golem!'")).stdout);
+  job.startWork(async (exe) => {
+    return String((await exe.run("echo -n 'Hello Golem!'")).stdout);
   });
   return job.id;
 }

diff --git a/examples/experimental/job/waitForResults.ts b/examples/experimental/job/waitForResults.ts
@@ -40,8 +40,8 @@ async function main() {
     console.log("Job ended event emitted");
   });
 
-  job.startWork(async (ctx) => {
-    return String((await ctx.run("echo -n 'Hello Golem!'")).stdout);
+  job.startWork(async (exe) => {
+    return String((await exe.run("echo -n 'Hello Golem!'")).stdout);
   });
 
   const result = await job.waitForResult();

diff --git a/examples/package.json b/examples/package.json
@@ -16,6 +16,7 @@
     "advanced-payment-filters": "tsx advanced/payment-filters.ts",
     "advanced-proposal-filters": "tsx advanced/proposal-filter.ts",
     "advanced-proposal-predefined-filter": "tsx advanced/proposal-predefined-filter.ts",
+    "advanced-setup-and-teardown": "tsx advanced/setup-and-teardown.ts",
     "local-image": "tsx advanced/local-image/serveLocalGvmi.ts",
     "deployment": "tsx experimental/deployment/new-api.ts",
     "market-scan": "tsx market/scan.ts",

diff --git a/examples/web/hello.html b/examples/web/hello.html
@@ -114,7 +114,7 @@ <h3>Results</h3>
           appendResults("Establishing a connection to the Golem Network");
           await glm.connect();
           appendResults("Request for renting a provider machine");
-          const rental = await glm.oneOf(order);
+          const rental = await glm.oneOf({ order });
           await rental
             .getExeUnit()
             .then(async (exe) =>

diff --git a/package.json b/package.json
@@ -38,7 +38,7 @@
     "docs": "typedoc src/ --plugin typedoc-plugin-merge-modules --plugin typedoc-theme-hierarchy",
     "docs:md": "typedoc src/ --plugin typedoc-plugin-markdown --plugin .docs/typedoc-frontmatter-theme.cjs --hideBreadcrumbs true && node .docs/summary-generator.cjs",
     "test": "npm run test:unit && npm run test:e2e",
-    "test:unit": "jest --config tests/unit/jest.config.json",
+    "test:unit": "jest --config tests/jest.config.json",
     "test:e2e": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --config tests/e2e/jest.config.json tests/e2e/**.spec.ts --runInBand --forceExit",
     "test:cypress": "cypress run",
     "test:examples": "tsx tests/examples/examples.test.ts",