Extract process adapter, introduce AdapterManager

packages/adapter-process/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+    ignorePatterns: [".eslintrc.js"],
+    parserOptions:{
+        project: "./tsconfig.json",
+        tsconfigRootDir: __dirname
+    }
+};

packages/adapter-process/.nycrc.json

@@ -0,0 +1,13 @@
+{
+    "all": true,
+    "include": [
+      "dist/**/*"
+    ],
+    "exclude": [
+        "**/*.spec.ts"
+    ],
+    "reporter": [
+        "lcovonly",
+        "text"
+    ]
+}

packages/adapter-process/README.md

@@ -0,0 +1,61 @@
+# Scramjet Transform Hub Adapters
+
+This module holds two types of adapters utilized by Scramjet Transform Hub: Instance Adapter and Sequence Adapter. These Adapters allows for running the Sequence identification and Instance execution in two basic modes: as a non containerized standalone processes or in a Docker container.
+
+The adapter provides two main exports:
+
+* [DockerSequenceAdapter](https://github.com/scramjetorg/transform-hub/tree/HEAD/packages/adapters/src/docker-sequence-adapter.ts) - An adapter for preparing Sequence to be run in Docker container.
+* [DockerInstanceAdapter](https://github.com/scramjetorg/transform-hub/tree/HEAD/packages/adapters/src/docker-instance-adapter.ts) - An adapter for running Instance by Runner executed in Docker container.
+
+## Docs
+
+See the code documentation here: [scramjetorg/transform-hub/docs/adapters/modules.md](https://github.com/scramjetorg/transform-hub/tree/HEAD/docs/adapters/modules.md)
+
+## Scramjet Transform Hub
+
+This package is part of [Scramjet Transform Hub](https://www.npmjs.org/package/@scramjet/sth).
+
+Scramjet Transform Hub is a deployment and execution platform. Once installed on a server, it will allow you to start your programs and keep them running on a remote machine. You will be able to start programs in the background or connect to them and see their output directly on your terminal. You will be able to pipe your local data to the program, as if it was running from your terminal. You can start your server in AWS, Google Cloud or Azure, start it on your local machine, install it on a Raspberry Pi or wherever else you'd like.
+
+## Use cases
+
+There's no limit what you can use it for. You want a stock checker? A chat bot? Maybe you'd like to automate your home? Retrieve sensor data? Maybe you have a lot of data and want to transfer and wrangle it? You have a database of cities and you'd like to enrich your data? You do machine learning and you want to train your set while the data is fetched in real time? Hey, you want to use it for something else and ask us if that's a good use? Ask us [via email](mailto:get@scramjet.org) or hop on our [Scramjet Discord](https://scr.je/join-community-mg1)!
+
+## Some important links
+
+* Scramjet, the company behind [Transform Hub](https://scramjet.org)
+* The [Scramjet Framework - functional reactive stream processing framework](https://framework.scramjet.org)
+* The [Transform Hub repo on github](https://github.com/scramjetorg/transform-hub)
+* You can see the [Scramjet Transform Hub API docs here](https://github.com/scramjetorg/transform-hub/tree/HEAD/docs/api-client/README.md)
+* You can see the [CLI documentation here](https://github.com/scramjetorg/transform-hub/tree/HEAD/packages/cli/README.md), but `si help` should also be quite effective.
+* Don't forget to ⭐ this repo if you like it, `subscribe` to releases and keep visiting us for new versions and updates.
+* You can [open an issue - file a bug report or a feature request here](https://github.com/scramjetorg/transform-hub/issues/new/choose)
+
+## License and contributions
+
+This module is licensed under AGPL-3.0 license.
+
+The Scramjet Transform Hub project is dual-licensed under the AGPL-3.0 and MIT licenses. Parts of the project that are linked with your programs are MIT licensed, the rest is AGPL.
+
+## Contributions
+
+We accept valid contributions and we will be publishing a more specific project roadmap so contributors can propose features and also help us implement them. We kindly ask you that contributed commits are Signed-Off `git commit --sign-off`.
+
+We provide support for contributors via test cases. If you expect a certain type of workflow to be officially supported, please specify and implement a test case in `Gherkin` format in `bdd` directory and include it in your pull request. More info about our BDD test you will find [here](https://github.com/scramjetorg/transform-hub/tree/HEAD/bdd/README.md).
+
+### Help wanted 👩‍🎓🧑👱‍♀️
+
+The project need's your help! There's lots of work to do and we have a lot of plans. If you want to help and be part of the Scramjet team, please reach out to us, [on discord](https://scr.je/join-community-mg1) or email us: [opensource@scramjet.org](mailto:opensource@scramjet.org).
+
+### Donation 💸
+
+Do you like this project? It helped you to reduce time spent on delivering your solution? You are welcome to buy us a coffee ☕ Thanks a lot! 😉
+
+[You can sponsor us on github](https://github.com/sponsors/scramjetorg)
+
+* There's also a Paypal donation link if you prefer that:
+
+[![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=7F7V65C43EBMW)
+
+
+

packages/adapter-process/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@scramjet/adapter-process",
+  "version": "0.34.4",
+  "description": "This package is part of Scramjet Transform Hub. This module holds the docker adapters utilized by Scramjet Transform Hub",
+  "main": "./src/index.ts",
+  "scripts": {
+    "start": "ts-node ./src/index",
+    "build": "../../scripts/build-all.js --config-name=tsconfig.build.json --copy-dist",
+    "build:docs": "typedoc",
+    "clean": "rm -rf ./dist .bic_cache",
+    "cloc": "cloc src --fullpath --include-lang TypeScript --not-match-d \"(node_modules|test|dist|bdd)\" --by-percent cm",
+    "test": "nyc ava",
+    "prepack": "node ../../scripts/publish.js"
+  },
+  "author": "Scramjet <open-source@scramjet.org>",
+  "license": "AGPL-3.0",
+  "dependencies": {
+    "@scramjet/model": "^0.34.4",
+    "@scramjet/obj-logger": "^0.34.4",
+    "@scramjet/pre-runner": "^0.34.4",
+    "@scramjet/python-runner": "^0.34.4",
+    "@scramjet/runner": "^0.34.4",
+    "@scramjet/sth-config": "^0.34.4",
+    "@scramjet/symbols": "^0.34.4",
+    "@scramjet/utility": "^0.34.4",
+    "@scramjet/adapters-utils": "^0.34.4",
+    "scramjet": "^4.36.9",
+    "shell-escape": "^0.2.0",
+    "systeminformation": "^5.12.7",
+    "ts.data.json": "^2.2.0"
+  },
+  "devDependencies": {
+    "@scramjet/types": "^0.34.4",
+    "@types/js-yaml": "4.0.5",
+    "@types/node": "15.12.5",
+    "@types/request": "2.48.8",
+    "@types/shell-escape": "^0.2.1",
+    "@types/ws": "8.5.3",
+    "ava": "^3.15.0",
+    "ts-node": "^10.9.1",
+    "typedoc": "^0.23.17",
+    "typedoc-plugin-markdown": "^3.13.6",
+    "typescript": "~4.7.4"
+  },
+  "ava": {
+    "extensions": [
+      "ts"
+    ],
+    "files": [
+      "**/*.spec.ts"
+    ],
+    "require": [
+      "ts-node/register"
+    ]
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/scramjetorg/transform-hub.git"
+  }
+}

packages/adapter-process/src/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Adapter module must provide SequenceAdapter, InstanceAdapter classes and name field.
+ */
+
+export { ProcessSequenceAdapter as SequenceAdapter }  from "./process-sequence-adapter";
+export { ProcessInstanceAdapter as InstanceAdapter } from "./process-instance-adapter";
+
+export const name = "process";

packages/adapter-process/src/process-instance-adapter.ts

@@ -0,0 +1,201 @@
+import { STHConfiguration,
+    ExitCode,
+    IComponent,
+    ILifeCycleAdapterMain,
+    ILifeCycleAdapterRun,
+    InstanceConfig,
+    InstanceLimits,
+    IObjectLogger,
+    MonitoringMessageData,
+    SequenceConfig
+} from "@scramjet/types";
+import { ObjLogger } from "@scramjet/obj-logger";
+import { streamToString } from "@scramjet/utility";
+import { getRunnerEnvVariables } from "@scramjet/adapters-utils";
+import { ChildProcess, spawn } from "child_process";
+
+import path from "path";
+
+const isTSNode = !!(process as any)[Symbol.for("ts-node.register.instance")];
+const gotPython = "\n                              _ \n __      _____  _ __  ___ ___| |\n \\ \\ /\\ / / _ \\| '_ \\/ __|_  / |\n  \\ V  V / (_) | | | \\__ \\/ /|_|\n   \\_/\\_/ \\___/|_| |_|___/___(_)  🐍\n";
+
+/**
+ * Adapter for running Instance by Runner executed in separate process.
+ */
+class ProcessInstanceAdapter implements
+    ILifeCycleAdapterMain,
+    ILifeCycleAdapterRun,
+    IComponent {
+    logger: IObjectLogger;
+    sthConfig: STHConfiguration;
+
+    private runnerProcess?: ChildProcess;
+    private crashLogStreams?: Promise<string[]>;
+    private _limits?: InstanceLimits = {};
+
+    get limits() { return this._limits || {} as InstanceLimits; }
+    private set limits(value: InstanceLimits) {
+        this._limits = value;
+        this.logger.warn("Limits are not yet supported in process runner");
+    }
+
+    constructor(config: STHConfiguration) {
+        this.logger = new ObjLogger(this);
+        this.sthConfig = config;
+    }
+
+    async init(): Promise<void> {
+        // noop
+    }
+
+    async stats(msg: MonitoringMessageData): Promise<MonitoringMessageData> {
+        const { runnerProcess } = this;
+
+        if (!runnerProcess) {
+            // Runner process not initialized yet
+            return msg;
+        }
+
+        return {
+            // @TODO: Provide stats and limits
+            ...msg,
+            processId: this.runnerProcess?.pid
+        };
+    }
+
+    getRunnerCmd(config: SequenceConfig) {
+        const engines = Object.keys(config.engines);
+        let debugFlags: string[] = [];
+
+        if (engines.length > 1) {
+            throw new Error("Incorrect config passed to SequenceConfig," +
+                "'engines' field can't contain more than one element");
+        }
+
+        if ("python3" in config.engines) {
+            this.logger.trace(gotPython);
+            const runnerPath = path.resolve(__dirname, require.resolve("@scramjet/python-runner"));
+
+            if (this.sthConfig.debug)
+                debugFlags = ["-m", "pdb", "-c", "continue"];
+
+            return [
+                "/usr/bin/env",
+                "python3",
+                ...debugFlags,
+                path.resolve(__dirname, runnerPath),
+                "./python-runner-startup.log",
+            ];
+        }
+        if (this.sthConfig.debug)
+            debugFlags = ["--inspect-brk=9229"];
+
+        return [
+            isTSNode ? "ts-node" : process.execPath,
+            ...debugFlags,
+            path.resolve(__dirname,
+                process.env.ESBUILD
+                    ? "../../runner/bin/start-runner.js"
+                    : require.resolve("@scramjet/runner")
+            )
+        ];
+    }
+
+    getPythonpath(sequenceDir: string) {
+        // This is for running from source. When the package is built, dependencies
+        // are installed next to runner.py script (rather than in __pypackages__),
+        // but that directory is automatically included in PYTHONPATH.
+        let pythonpath = path.resolve(
+            __dirname, require.resolve("@scramjet/python-runner"), "../__pypackages__"
+        );
+
+        if (process.env.PYTHONPATH) pythonpath += `:${process.env.PYTHONPATH}`;
+
+        pythonpath += `:${sequenceDir}/__pypackages__`;
+
+        return pythonpath;
+    }
+
+    async run(config: InstanceConfig, instancesServerPort: number, instanceId: string): Promise<ExitCode> {
+        if (config.type !== "process") {
+            throw new Error("Process instance adapter run with invalid runner config");
+        }
+
+        this.logger.info("Instance preparation done");
+
+        this.logger.trace("Starting Runner", config.id);
+
+        const runnerCommand = this.getRunnerCmd(config);
+        const sequencePath = path.join(
+            config.sequenceDir,
+            config.entrypointPath
+        );
+        const env = getRunnerEnvVariables({
+            sequencePath,
+            instancesServerHost: "127.0.0.1",
+            instancesServerPort,
+            instanceId,
+            pipesPath: ""
+        }, {
+            PYTHONPATH: this.getPythonpath(config.sequenceDir),
+        });
+
+        this.logger.debug("Spawning Runner process with command", runnerCommand);
+        this.logger.trace("Runner process environment", env);
+
+        const runnerProcess = spawn(runnerCommand[0], runnerCommand.slice(1), { env });
+
+        this.crashLogStreams = Promise.all([runnerProcess.stdout, runnerProcess.stderr].map(streamToString));
+
+        this.logger.trace("Runner process is running", runnerProcess.pid);
+
+        this.runnerProcess = runnerProcess;
+
+        const [statusCode, signal] = await new Promise<[number | null, NodeJS.Signals | null]>(
+            (res) => runnerProcess.on("exit", (code, sig) => res([code, sig]))
+        );
+
+        this.logger.trace("Runner process exited", runnerProcess.pid);
+
+        if (statusCode === null) {
+            this.logger.warn("Runner was killed by a signal, and didn't return a status code", signal);
+
+            // Probably SIGIKLL
+            return 137;
+        }
+
+        if (statusCode > 0) {
+            this.logger.debug("Process returned non-zero status code", statusCode);
+        }
+
+        return statusCode;
+    }
+
+    /**
+     * Performs cleanup after Runner end.
+     * Removes fifos used to communication with runner.
+     */
+    async cleanup(): Promise<void> {
+        //noop
+    }
+
+    // @ts-ignore
+    monitorRate(_rps: number): this {
+        /** ignore */
+    }
+
+    /**
+     * Forcefully stops Runner process.
+     */
+    async remove() {
+        this.runnerProcess?.kill();
+    }
+
+    async getCrashLog(): Promise<string[]> {
+        if (!this.crashLogStreams) return [];
+
+        return this.crashLogStreams;
+    }
+}
+
+export { ProcessInstanceAdapter  };