From 5e5c727a35c77ad76958108f4b548a48d524b2e1 Mon Sep 17 00:00:00 2001 From: patuwwy Date: Thu, 29 Jun 2023 11:29:00 +0000 Subject: [PATCH] adapter-k8s, remove old adapters package --- .../{adapters => adapter-k8s}/.eslintrc.js | 0 .../{adapters => adapter-k8s}/.nycrc.json | 0 packages/{adapters => adapter-k8s}/README.md | 0 .../{adapters => adapter-k8s}/package.json | 7 +- packages/adapter-k8s/src/index.ts | 11 + .../src/kubernetes-client-adapter.ts | 0 .../src/kubernetes-config-decoder.ts | 0 .../src/kubernetes-instance-adapter.ts | 2 +- .../src/kubernetes-sequence-adapter.ts | 3 +- .../{adapters => adapter-k8s}/src/readme.mtpl | 6 +- .../test/pass.spec.ts | 0 .../tsconfig.build.json | 0 .../{adapters => adapter-k8s}/tsconfig.json | 0 .../adapters/src/docker-instance-adapter.ts | 285 ------------- packages/adapters/src/docker-networking.ts | 41 -- .../adapters/src/docker-sequence-adapter.ts | 299 ------------- .../adapters/src/dockerode-docker-helper.ts | 398 ------------------ packages/adapters/src/get-instance-adapter.ts | 33 -- packages/adapters/src/get-runner-env.ts | 40 -- packages/adapters/src/get-sequence-adapter.ts | 30 -- packages/adapters/src/index.ts | 16 - .../src/initialize-runtime-adapters.ts | 25 -- .../adapters/src/process-instance-adapter.ts | 200 --------- .../adapters/src/process-sequence-adapter.ts | 165 -------- packages/adapters/src/types.ts | 330 --------------- packages/adapters/src/utils.ts | 16 - .../src/validate-sequence-package-json.ts | 52 --- packages/host/src/lib/adapter-manager.ts | 4 +- packages/host/src/lib/host.ts | 4 +- yarn.lock | 45 +- 30 files changed, 46 insertions(+), 1966 deletions(-) rename packages/{adapters => adapter-k8s}/.eslintrc.js (100%) rename packages/{adapters => adapter-k8s}/.nycrc.json (100%) rename packages/{adapters => adapter-k8s}/README.md (100%) rename packages/{adapters => adapter-k8s}/package.json (94%) create mode 100644 packages/adapter-k8s/src/index.ts rename packages/{adapters => adapter-k8s}/src/kubernetes-client-adapter.ts (100%) rename packages/{adapters => adapter-k8s}/src/kubernetes-config-decoder.ts (100%) rename packages/{adapters => adapter-k8s}/src/kubernetes-instance-adapter.ts (99%) rename packages/{adapters => adapter-k8s}/src/kubernetes-sequence-adapter.ts (97%) rename packages/{adapters => adapter-k8s}/src/readme.mtpl (50%) rename packages/{adapters => adapter-k8s}/test/pass.spec.ts (100%) rename packages/{adapters => adapter-k8s}/tsconfig.build.json (100%) rename packages/{adapters => adapter-k8s}/tsconfig.json (100%) delete mode 100644 packages/adapters/src/docker-instance-adapter.ts delete mode 100644 packages/adapters/src/docker-networking.ts delete mode 100644 packages/adapters/src/docker-sequence-adapter.ts delete mode 100644 packages/adapters/src/dockerode-docker-helper.ts delete mode 100644 packages/adapters/src/get-instance-adapter.ts delete mode 100644 packages/adapters/src/get-runner-env.ts delete mode 100644 packages/adapters/src/get-sequence-adapter.ts delete mode 100644 packages/adapters/src/index.ts delete mode 100644 packages/adapters/src/initialize-runtime-adapters.ts delete mode 100644 packages/adapters/src/process-instance-adapter.ts delete mode 100644 packages/adapters/src/process-sequence-adapter.ts delete mode 100644 packages/adapters/src/types.ts delete mode 100644 packages/adapters/src/utils.ts delete mode 100644 packages/adapters/src/validate-sequence-package-json.ts diff --git a/packages/adapters/.eslintrc.js b/packages/adapter-k8s/.eslintrc.js similarity index 100% rename from packages/adapters/.eslintrc.js rename to packages/adapter-k8s/.eslintrc.js diff --git a/packages/adapters/.nycrc.json b/packages/adapter-k8s/.nycrc.json similarity index 100% rename from packages/adapters/.nycrc.json rename to packages/adapter-k8s/.nycrc.json diff --git a/packages/adapters/README.md b/packages/adapter-k8s/README.md similarity index 100% rename from packages/adapters/README.md rename to packages/adapter-k8s/README.md diff --git a/packages/adapters/package.json b/packages/adapter-k8s/package.json similarity index 94% rename from packages/adapters/package.json rename to packages/adapter-k8s/package.json index d70625b25..1c2f03050 100644 --- a/packages/adapters/package.json +++ b/packages/adapter-k8s/package.json @@ -1,5 +1,5 @@ { - "name": "@scramjet/adapters", + "name": "@scramjet/adapter-k8s", "version": "0.34.4", "description": "This package is part of Scramjet Transform Hub. This module holds the docker adapters utilized by Scramjet Transform Hub", "main": "./src/index.ts", @@ -15,7 +15,6 @@ "author": "Scramjet ", "license": "AGPL-3.0", "dependencies": { - "@kubernetes/client-node": "^0.17.1", "@scramjet/model": "^0.34.4", "@scramjet/obj-logger": "^0.34.4", "@scramjet/pre-runner": "^0.34.4", @@ -24,7 +23,8 @@ "@scramjet/sth-config": "^0.34.4", "@scramjet/symbols": "^0.34.4", "@scramjet/utility": "^0.34.4", - "dockerode": "^3.3.4", + "@scramjet/adapters-utils": "^0.34.4", + "@kubernetes/client-node": "^0.17.1", "scramjet": "^4.36.9", "shell-escape": "^0.2.0", "systeminformation": "^5.12.7", @@ -32,7 +32,6 @@ }, "devDependencies": { "@scramjet/types": "^0.34.4", - "@types/dockerode": "<=3.3.3", "@types/js-yaml": "4.0.5", "@types/node": "15.12.5", "@types/request": "2.48.8", diff --git a/packages/adapter-k8s/src/index.ts b/packages/adapter-k8s/src/index.ts new file mode 100644 index 000000000..1e8f17476 --- /dev/null +++ b/packages/adapter-k8s/src/index.ts @@ -0,0 +1,11 @@ +/** + * Adapter module must provide SequenceAdapter, InstanceAdapter classes, init method and name field. + */ +export { KubernetesSequenceAdapter as SequenceAdapter } from "./kubernetes-sequence-adapter"; +export { KubernetesInstanceAdapter as InstanceAdapter } from "./kubernetes-instance-adapter"; + +export const init = (..._args: any[]) => { + return true; +}; + +export const name = "kubernetes"; diff --git a/packages/adapters/src/kubernetes-client-adapter.ts b/packages/adapter-k8s/src/kubernetes-client-adapter.ts similarity index 100% rename from packages/adapters/src/kubernetes-client-adapter.ts rename to packages/adapter-k8s/src/kubernetes-client-adapter.ts diff --git a/packages/adapters/src/kubernetes-config-decoder.ts b/packages/adapter-k8s/src/kubernetes-config-decoder.ts similarity index 100% rename from packages/adapters/src/kubernetes-config-decoder.ts rename to packages/adapter-k8s/src/kubernetes-config-decoder.ts diff --git a/packages/adapters/src/kubernetes-instance-adapter.ts b/packages/adapter-k8s/src/kubernetes-instance-adapter.ts similarity index 99% rename from packages/adapters/src/kubernetes-instance-adapter.ts rename to packages/adapter-k8s/src/kubernetes-instance-adapter.ts index e91ff7bde..e403de393 100644 --- a/packages/adapters/src/kubernetes-instance-adapter.ts +++ b/packages/adapter-k8s/src/kubernetes-instance-adapter.ts @@ -17,7 +17,7 @@ import { ObjLogger } from "@scramjet/obj-logger"; import { createReadStream } from "fs"; import { KubernetesClientAdapter } from "./kubernetes-client-adapter"; import { adapterConfigDecoder } from "./kubernetes-config-decoder"; -import { getRunnerEnvEntries } from "./get-runner-env"; +import { getRunnerEnvEntries } from "@scramjet/adapters-utils"; import { PassThrough } from "stream"; import { RunnerExitCode } from "@scramjet/symbols"; diff --git a/packages/adapters/src/kubernetes-sequence-adapter.ts b/packages/adapter-k8s/src/kubernetes-sequence-adapter.ts similarity index 97% rename from packages/adapters/src/kubernetes-sequence-adapter.ts rename to packages/adapter-k8s/src/kubernetes-sequence-adapter.ts index 5fc3d97d4..e7b620be8 100644 --- a/packages/adapters/src/kubernetes-sequence-adapter.ts +++ b/packages/adapter-k8s/src/kubernetes-sequence-adapter.ts @@ -13,9 +13,8 @@ import fs from "fs/promises"; import path from "path"; import { exec } from "child_process"; import { isDefined, readStreamedJSON } from "@scramjet/utility"; -import { sequencePackageJSONDecoder } from "./validate-sequence-package-json"; +import { detectLanguage, sequencePackageJSONDecoder } from "@scramjet/adapters-utils"; import { adapterConfigDecoder } from "./kubernetes-config-decoder"; -import { detectLanguage } from "./utils"; /** * Returns existing Sequence configuration. diff --git a/packages/adapters/src/readme.mtpl b/packages/adapter-k8s/src/readme.mtpl similarity index 50% rename from packages/adapters/src/readme.mtpl rename to packages/adapter-k8s/src/readme.mtpl index ee86c0219..237e68061 100644 --- a/packages/adapters/src/readme.mtpl +++ b/packages/adapter-k8s/src/readme.mtpl @@ -1,11 +1,11 @@ # Scramjet Transform Hub Adapters -This module holds two types of adapters utilized by Scramjet Transform Hub: Instance Adapter and Sequence Adapter. These Adapters allows for running the Sequence identification and Instance execution in two basic modes: as a non containerized standalone processes or in a Docker container. +This module holds two types of adapters utilized by Scramjet Transform Hub: Instance Adapter and Sequence Adapter. These Adapters allows for running the Sequence identification and Instance execution in standalone processes. The adapter provides two main exports: -* [DockerSequenceAdapter](./src/docker-sequence-adapter.ts) - An adapter for preparing Sequence to be run in Docker container. -* [DockerInstanceAdapter](./src/docker-instance-adapter.ts) - An adapter for running Instance by Runner executed in Docker container. +* [SequenceAdapter](./src/process-sequence-adapter.ts) - An adapter for preparing Sequence to be run in subprocess. +* [InstanceAdapter](./src/process-instance-adapter.ts) - An adapter for running Instance by Runner executed in subprocess. ## Docs diff --git a/packages/adapters/test/pass.spec.ts b/packages/adapter-k8s/test/pass.spec.ts similarity index 100% rename from packages/adapters/test/pass.spec.ts rename to packages/adapter-k8s/test/pass.spec.ts diff --git a/packages/adapters/tsconfig.build.json b/packages/adapter-k8s/tsconfig.build.json similarity index 100% rename from packages/adapters/tsconfig.build.json rename to packages/adapter-k8s/tsconfig.build.json diff --git a/packages/adapters/tsconfig.json b/packages/adapter-k8s/tsconfig.json similarity index 100% rename from packages/adapters/tsconfig.json rename to packages/adapter-k8s/tsconfig.json diff --git a/packages/adapters/src/docker-instance-adapter.ts b/packages/adapters/src/docker-instance-adapter.ts deleted file mode 100644 index 760aaa719..000000000 --- a/packages/adapters/src/docker-instance-adapter.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { InstanceAdapterError } from "@scramjet/model"; -import { - ContainerConfiguration, - ContainerConfigurationWithExposedPorts, - ExitCode, - IComponent, - ILifeCycleAdapterMain, - ILifeCycleAdapterRun, - IObjectLogger, - MonitoringMessageData, - InstanceConfig, - RunnerContainerConfiguration, - InstanceLimits, - STHConfiguration, -} from "@scramjet/types"; -import path from "path"; -import { DockerodeDockerHelper } from "./dockerode-docker-helper"; -import { DockerAdapterResources, DockerAdapterRunPortsConfig, DockerAdapterVolumeConfig, IDockerHelper } from "./types"; -import { FreePortsFinder, defer, streamToString } from "@scramjet/utility"; -import { STH_DOCKER_NETWORK, isHostSpawnedInDockerContainer, getHostname } from "./docker-networking"; -import { ObjLogger } from "@scramjet/obj-logger"; -import { getRunnerEnvEntries } from "./get-runner-env"; -import { Readable } from "stream"; - -/** - * Adapter for running Instance by Runner executed in Docker container. - */ -class DockerInstanceAdapter implements -ILifeCycleAdapterMain, -ILifeCycleAdapterRun, -IComponent { - private dockerHelper: IDockerHelper; - private _limits?: InstanceLimits = {}; - private resources: DockerAdapterResources = {}; - id: string = ""; - - logger: IObjectLogger; - - crashLogStreams?: Promise; - - get limits() { return this._limits || {} as InstanceLimits; } - private set limits(value: InstanceLimits) { this._limits = value; } - - constructor(_sthConfig: STHConfiguration, id: string = "") { - this.dockerHelper = new DockerodeDockerHelper(); - - this.logger = new ObjLogger(this, { id }); - this.dockerHelper.logger.updateBaseLog({ id }); - this.dockerHelper.logger.pipe(this.logger); - } - - async init(): Promise { - /** ignore */ - } - - /** - * Finds free port for every port requested in Sequence configuration and returns map of assigned ports. - * - * @param {string[]} declaredPorts Ports declared in sequence config. - * @param {ContainerConfigurationWithExposedPorts} containerConfig Container configuration - * extended with configuration for ports exposing. - * @param {boolean} [exposed=false] Defines configuration output type. Exposed ports when true or port bindings. - * - * @returns Promise resolving with map of ports mapping. - */ - private async preparePortBindingsConfig( - declaredPorts: string[], - containerConfig: ContainerConfiguration & ContainerConfigurationWithExposedPorts, - exposed: boolean = false): Promise<{ [key: string]: string; }> { - if (declaredPorts.every(entry => (/^\d{3,5}\/(tcp|udp)$/).test(entry))) { - const freePorts = exposed ? [] : await FreePortsFinder.getPorts( - declaredPorts.length, ...containerConfig.exposePortsRange - ); - - return declaredPorts.reduce((obj: { [ key: string ]: any }, entry: string) => { - if (entry) { - const { port, protocol } = entry.match(/^(?\d{3,5})\/(?(tcp|udp))$/)?.groups as { port: string, protocol: string }; - - obj[`${port}/${protocol}`] = exposed ? {} : [{ HostIp: containerConfig.hostIp, HostPort: freePorts?.pop()?.toString() }]; - } - - return obj; - }, {}); - } - - throw new InstanceAdapterError("INVALID_CONFIGURATION", "Incorrect ports configuration provided."); - } - - /** - * Prepares configuration for expose/bind ports from Docker container. - * - * @param {string[]} ports Ports requested to be accessible from container. - * @param {RunnerContainerConfiguration} containerConfig Runner container configuration. - * - * @returns Configuration for exposing and binding ports in Docker container. - */ - private async getPortsConfig( - ports: string[], containerConfig: RunnerContainerConfiguration - ): Promise { - const [ExposedPorts, PortBindings] = await Promise.all([ - this.preparePortBindingsConfig(ports, containerConfig, true), - this.preparePortBindingsConfig(ports, containerConfig, false) - ]); - - return { ExposedPorts, PortBindings }; - } - - /** - * Returns objects with statistics of docker container with running instance. - * - * @param {MonitoringMessageData} msg Message to be included in statistics message. - * @returns {Promise} Promise resolved with container statistics. - */ - async stats(msg: MonitoringMessageData): Promise { - if (this.resources.containerId) { - const stats = await this.dockerHelper.stats(this.resources.containerId)!; - - return { - cpuTotalUsage: stats.cpu_stats?.cpu_usage?.total_usage, - healthy: msg.healthy, - limit: stats.memory_stats?.limit, - memoryMaxUsage: stats.memory_stats?.max_usage, - memoryUsage: stats.memory_stats?.usage, - networkRx: stats.networks?.eth0?.rx_bytes, - networkTx: stats.networks?.eth0?.tx_bytes, - containerId: this.resources.containerId - }; - } - - return msg; - } - - private async getNetworkSetup(): Promise<{ network: string, host: string }> { - const interfaces = await this.dockerHelper.listNetworks(); - const sthDockerNetwork = interfaces.find(net => net.Name === STH_DOCKER_NETWORK); - - if (!sthDockerNetwork) { - // STH docker network should be created in Host initialization - throw new Error(`Couldn't find sth docker network: ${sthDockerNetwork}`); - } - - if (await isHostSpawnedInDockerContainer()) { - const hostname = getHostname(); - - // If Transform Hub runs in Docker container - // then this container should be connected to STH docker network in Host initialization - - this.logger.debug("Runner will connect to STH container with hostname", hostname); - - return { - network: STH_DOCKER_NETWORK, - host: hostname, - }; - } - // otherwise STH runs on Host OS so we Runner can just connect to the Gateway - const sthNetworkGateway = sthDockerNetwork?.IPAM?.Config?.[0]?.Gateway; - - if (!sthNetworkGateway) { - throw new Error(`Couldn't determine gateway for ${STH_DOCKER_NETWORK}`); - } - - this.logger.debug("Runner will connect to STH on host OS using gateway", sthNetworkGateway); - - return { - network: STH_DOCKER_NETWORK, - host: sthNetworkGateway - }; - } - - // eslint-disable-next-line complexity - async run(config: InstanceConfig, instancesServerPort: number, instanceId: string): Promise { - if (config.type !== "docker") { - throw new Error("Docker instance adapter run with invalid runner config"); - } - - this.limits = config.limits; - - this.resources.ports = - config.config?.ports ? await this.getPortsConfig(config.config.ports, config.container) : undefined; - - config.container.maxMem = config.limits.memory || config.container.maxMem; - - this.logger.info("Instance preparation done for config", config); - - const extraVolumes: DockerAdapterVolumeConfig[] = []; - - const networkSetup = await this.getNetworkSetup(); - - const envs = getRunnerEnvEntries({ - sequencePath: path.join(config.sequenceDir, config.entrypointPath), - instancesServerPort, - instancesServerHost: networkSetup.host, - instanceId, - pipesPath: "" - }).map(([k, v]) => `${k}=${v}`); - - this.logger.debug("Runner will start with envs", envs); - - const { containerId, streams } = await this.dockerHelper.run({ - imageName: config.container.image, - volumes: [ - ...extraVolumes, - { mountPoint: config.sequenceDir, volume: config.id, writeable: false } - ], - labels: { - "scramjet.sequence.name": config.name - }, - ports: this.resources.ports, - publishAllPorts: true, - envs, - autoRemove: true, - maxMem: config.container.maxMem, - networkMode: networkSetup.network - }); - - this.crashLogStreams = Promise.all(([streams.stdout, streams.stderr] as Readable[]).map(streamToString)); - - this.resources.containerId = containerId; - - this.logger.trace("Container is running", containerId); - - try { - const { statusCode } = await this.dockerHelper.wait(containerId); - - this.logger.debug("Container exited", statusCode); - - if (statusCode > 0) { - throw new InstanceAdapterError("RUNNER_NON_ZERO_EXITCODE", { statusCode }); - } else { - return 0; - } - } catch (error: any) { - if (error instanceof InstanceAdapterError && error.code === "RUNNER_NON_ZERO_EXITCODE" && error.data.statusCode) { - this.logger.debug("Container returned non-zero status code", error.data.statusCode); - - return error.data.statusCode; - } - - this.logger.debug("Container errored", error); - - throw error; - } - } - - /** - * Performs cleanup after container close. - * Removes volume used by sequence and fifos used to communication with runner. - */ - async cleanup(): Promise { - if (this.resources.volumeId) { - this.logger.debug("Volume will be removed in 60 sec"); - - await defer(60000); // @TODO: one sec? - await this.dockerHelper.removeVolume(this.resources.volumeId); - - this.logger.debug("Volume removed"); - } - } - - // @ts-ignore - monitorRate(_rps: number): this { - /** ignore */ - } - - /** - * Forcefully stops Runner container. - */ - async remove() { - if (this.resources.containerId) { - this.logger.debug("Forcefully stopping container", this.resources.containerId); - - await this.dockerHelper.stopContainer(this.resources.containerId); - - this.logger.debug("Container removed"); - } - } - - async getCrashLog(): Promise { - if (!this.crashLogStreams) return []; - - return this.crashLogStreams; - } -} - -export { DockerInstanceAdapter }; diff --git a/packages/adapters/src/docker-networking.ts b/packages/adapters/src/docker-networking.ts deleted file mode 100644 index 7533f80ef..000000000 --- a/packages/adapters/src/docker-networking.ts +++ /dev/null @@ -1,41 +0,0 @@ -import fs from "fs/promises"; -import os from "os"; -import { IDockerHelper } from "./types"; - -export const isHostSpawnedInDockerContainer = async () => await fs.access("/.dockerenv").then(() => true, () => false); - -export const getHostname = () => os.hostname(); - -export const STH_DOCKER_NETWORK = "transformhub0"; - -// @TODO this could be encapsulated into IInstanceAdapter for doing something on Transform Hub launch -export async function setupDockerNetworking(dockerHelper: IDockerHelper) { - const networkExists = await dockerHelper.inspectNetwork(STH_DOCKER_NETWORK).then(() => true, () => false); - - if (!networkExists) { - await dockerHelper.createNetwork({ - name: STH_DOCKER_NETWORK, - driver: "bridge", - options: { - "com.docker.network.bridge.host_binding_ipv4":"0.0.0.0", - "com.docker.network.bridge.enable_ip_masquerade":"true", - "com.docker.network.bridge.enable_icc":"true", - "com.docker.network.driver.mtu":"1500" - } - }); - } - - if (await isHostSpawnedInDockerContainer()) { - const { containers } = await dockerHelper.inspectNetwork(STH_DOCKER_NETWORK); - - const hostname = getHostname(); - - const isHostConnected = !!Object.entries(containers).find( - ([id, { Name }]: [string, any]) => id.startsWith(hostname) || Name === hostname - ); - - if (!isHostConnected) { - await dockerHelper.connectToNetwork(STH_DOCKER_NETWORK, hostname); - } - } -} diff --git a/packages/adapters/src/docker-sequence-adapter.ts b/packages/adapters/src/docker-sequence-adapter.ts deleted file mode 100644 index 3bec26f37..000000000 --- a/packages/adapters/src/docker-sequence-adapter.ts +++ /dev/null @@ -1,299 +0,0 @@ -import { SequenceAdapterError } from "@scramjet/model"; -import { - ISequenceAdapter, - SequenceConfig, - DockerSequenceConfig, - IObjectLogger, - DockerAdapterConfiguration -} from "@scramjet/types"; -import { Readable } from "stream"; -import { appendFile } from "fs"; -import { DockerodeDockerHelper } from "./dockerode-docker-helper"; -import { - DockerAdapterResources, - DockerAdapterRunResponse, - DockerAdapterStreams, - DockerVolume, - IDockerHelper -} from "./types"; -import { isDefined, readStreamedJSON } from "@scramjet/utility"; -import { ObjLogger } from "@scramjet/obj-logger"; -import { sequencePackageJSONDecoder } from "./validate-sequence-package-json"; -import { detectLanguage } from "./utils"; - -const PACKAGE_DIR = "/package"; - -/** - * Adapter for preparing Sequence to be run in Docker container. - */ -class DockerSequenceAdapter implements ISequenceAdapter { - private dockerHelper: IDockerHelper; - private resources: DockerAdapterResources = {}; - private config: DockerAdapterConfiguration; - - public name = "DockerSequenceAdapter"; - - /** - * Instance of class providing logging utilities. - */ - logger: IObjectLogger; - - constructor(config: DockerAdapterConfiguration) { - this.config = config; - this.logger = new ObjLogger(this.name); - - this.dockerHelper = new DockerodeDockerHelper(); - this.dockerHelper.logger.pipe(this.logger); - } - - /** - * Initializes adapter. - */ - async init(): Promise { - this.logger.trace("Initializing"); - - await this.fetch(this.config.prerunner.image); - - this.logger.info("Docker adapter initialized with options", { - "python3 runner image": this.config.runnerImages.python3, - "node runner image": this.config.runnerImages.node, - "prerunner image": this.config.prerunner.image - }); - } - - /** - * Pulls image from registry. - * - * @param {string} name Docker image name - */ - async fetch(name: string) { - await this.dockerHelper.pullImage(name, true); - } - - /** - * Finds existing Docker volumes containing sequences. - * - * @returns {Promise} Promise resolving to array of identified sequences. - */ - async list(): Promise { - this.logger.trace("Listing exiting sequences"); - - const potentialVolumes = await this.dockerHelper.listVolumes(); - - const configs = await Promise.all( - potentialVolumes - .map(volume => this.identifyOnly(volume)) - .map(configPromised => configPromised.catch(() => null)) - ); - - return configs.filter(isDefined); - } - - /** - * Identifies sequence existing on Docker volume. - * - * @param {string} volume Volume id. - * @returns {SequenceConfig} Sequence configuration or undefined if sequence cannot be identified. - */ - private async identifyOnly(volume: string): Promise { - this.logger.info("Attempting to identify volume", volume); - - try { - const { streams, wait } = await this.dockerHelper.run({ - imageName: this.config.prerunner?.image || "", - volumes: [{ mountPoint: PACKAGE_DIR, volume, writeable: true }], - command: ["/opt/transform-hub/identify.sh"], - autoRemove: true, - maxMem: this.config.prerunner?.maxMem || 0 - }); - - this.logger.debug("Identify started", volume, this.config.prerunner?.maxMem || 0); - - const ret = await this.parsePackage(streams, wait, volume); - - if (!ret.id) { - return undefined; - } - - this.logger.info("Identified image for volume", { volume, image: ret.container?.image }); - - return ret; - } catch (e: any) { - this.logger.error("Docker failed", e.message, volume); - - throw e; - } - } - - /** - * Unpacks and identifies sequence in Docker volume. - * This is the main adapter method creating new Docker volume and starting Prerunner - * with created volume mounted to unpack sequence on it. - * When Prerunner finishes, it will return JSON with sequence information. - * - * @param {Readable} stream Stream containing sequence to be identified. - * @param {string} id Id for the new docker volume where sequence will be stored. - * @param {boolean} override Removes previous sequence - * @returns {Promise} Promise resolving to sequence config. - */ - async identify(stream: Readable, id: string, override = false): Promise { - const volStart = new Date(); - - if (override) { - await this.dockerHelper.removeVolume(id); - } - - const volumeId = await this.createVolume(id); - - const volSecs = (new Date().getTime() - volStart.getTime()) / 1000; - - appendFile("timing-log.ndjson", JSON.stringify({ - operation: "creating volume", - volumeId: volumeId, - time: volSecs, - }) + "\n", () => {}); - - this.resources.volumeId = volumeId; - - this.logger.info(`Volume created in ${volSecs}s`, volumeId); - - let runResult: DockerAdapterRunResponse; - const prerunnerStart = new Date(); - - this.logger.debug("Starting PreRunner", this.config.prerunner); - - try { - runResult = await this.dockerHelper.run({ - imageName: this.config.prerunner.image || "", - volumes: [{ mountPoint: PACKAGE_DIR, volume: volumeId, writeable: true }], - autoRemove: true, - maxMem: this.config.prerunner.maxMem || 0 - }); - } catch (err: any) { - this.logger.error(err); - - throw new SequenceAdapterError("DOCKER_ERROR"); - } - - const startSecs = (new Date().getTime() - prerunnerStart.getTime()) / 1000; - - appendFile("timing-log.ndjson", JSON.stringify({ - operation: "starting pre-runner", - time: startSecs, - }) + "\n", () => {}); - - try { - const { streams, wait } = runResult; - - stream.pipe(streams.stdin); - - const config = await this.parsePackage(streams, wait, volumeId); - - await this.fetch(config.container.image); - - return config; - } catch (err: any) { - this.logger.error("Identify failed on volume", id); - if (err instanceof SequenceAdapterError) { - throw err; - } else { - throw new SequenceAdapterError("PRERUNNER_ERROR", err); - } - } - } - - /** - * Creates volume with provided id. - * - * @param {string} id Volume id. - * @returns {DockerVolume} Created volume. - */ - private async createVolume(id: string): Promise { - try { - return await this.dockerHelper.createVolume(id); - } catch (error: any) { - this.logger.error("Error creating volume", id); - - throw new SequenceAdapterError("DOCKER_ERROR", "Error creating volume"); - } - } - - /** - * Parses PreRunner output and returns sequence configuration. - * - * @param {DockerAdapterStreams} streams Docker container std streams. - * @param {Function} wait TBD - * @param {DockerVolume} volumeId Id of the volume where sequence is stored. - * @returns {Promise} Promise resolving to sequence configuration. - */ - // eslint-disable-next-line complexity - private async parsePackage( - streams: DockerAdapterStreams, - wait: Function, - volumeId: DockerVolume - ): Promise { - const parseStart = new Date(); - - const [preRunnerResult] = (await Promise.all([readStreamedJSON(streams.stdout as Readable), wait])) as any; - - const parseSecs = (new Date().getTime() - parseStart.getTime()) / 1000; - - appendFile("timing-log.ndjson", JSON.stringify({ - operation: "waiting for pre-runner", - time: parseSecs, - }) + "\n", () => {}); - - this.logger.debug("PreRunner response", preRunnerResult); - - if (preRunnerResult && preRunnerResult.error) { - this.logger.error("PreRunner failed", preRunnerResult.error); - - throw new SequenceAdapterError("PRERUNNER_ERROR", preRunnerResult.error); - } - - const validPackageJson = await sequencePackageJSONDecoder.decodeToPromise(preRunnerResult); - const engines = validPackageJson.engines ? { ...validPackageJson.engines } : {}; - const config = validPackageJson.scramjet?.config ? { ...validPackageJson.scramjet.config } : {}; - - const container = Object.assign({}, this.config.runner); - - container.image = "python3" in engines - ? this.config.runnerImages.python3 - : this.config.runnerImages.node; - - return { - type: "docker", - container, - name: validPackageJson.name || "", - version: validPackageJson.version || "", - engines, - config, - sequenceDir: PACKAGE_DIR, - entrypointPath: validPackageJson.main, - id: volumeId, - description: validPackageJson.description, - author: validPackageJson.author, - keywords: validPackageJson.keywords, - args: validPackageJson.args, - repository: validPackageJson.repository, - language: detectLanguage(validPackageJson) - }; - } - - /** - * Removes Docker volume used by Sequence. - * - * @param {SequenceConfig} config Sequence configuration. - */ - async remove(config: SequenceConfig) { - if (config.type !== "docker") { - throw new Error(`Incorrect SequenceConfig passed to DockerSequenceAdapter: ${config.type}`); - } - - await this.dockerHelper.removeVolume(config.id); - - this.logger.debug("Volume removed", config.id); - } -} - -export { DockerSequenceAdapter }; diff --git a/packages/adapters/src/dockerode-docker-helper.ts b/packages/adapters/src/dockerode-docker-helper.ts deleted file mode 100644 index d87351d8f..000000000 --- a/packages/adapters/src/dockerode-docker-helper.ts +++ /dev/null @@ -1,398 +0,0 @@ -import Dockerode from "dockerode"; -import { PassThrough } from "stream"; -import { appendFile } from "fs"; - -import { - DockerAdapterRunConfig, - DockerAdapterRunResponse, - DockerAdapterStreams, DockerAdapterVolumeConfig, - DockerAdapterWaitOptions, - DockerContainer, - IDockerHelper, DockerImage, DockerVolume, ExitData, - DockerCreateNetworkConfig, DockerNetwork -} from "./types"; -import { ObjLogger } from "@scramjet/obj-logger"; - -/** - * Configuration for volumes to be mounted to container. - */ -type DockerodeVolumeMountConfig = { - /** - * Directory in container where volume has to be mounted. - */ - Target: string, - - /** - * Volume name. - */ - Source: string, - - /** - * Mounting mode. - */ - Type: "volume" | "bind", - - /** - * Access mode. - */ - ReadOnly: boolean -} - -let _isDockerConfigured: boolean|undefined; - -async function isDockerConfigured() { - try { - await new Dockerode().info(); - _isDockerConfigured = true; - } catch (e) { - _isDockerConfigured = false; - } - - return _isDockerConfigured; -} - -/** - * Communicates with Docker using Dockerode library. - */ -export class DockerodeDockerHelper implements IDockerHelper { - public dockerode: Dockerode = new Dockerode(); - - logger = new ObjLogger(this); - - /** - * Translates DockerAdapterVolumeConfig to volumes configuration that Docker API can understand. - * - * @param {DockerAdapterVolumeConfig[]} volumeConfigs Volumes configuration. - * @returns {DockerodeVolumeMountConfig[]} Translated volumes configuration. - */ - translateVolumesConfig(volumeConfigs: DockerAdapterVolumeConfig[]): DockerodeVolumeMountConfig[] { - return volumeConfigs.map(cfg => { - if ("bind" in cfg) { - return { - Target: cfg.mountPoint, - Source: cfg.bind, - Type: "bind", - ReadOnly: !cfg.writeable - }; - } - - return { - Target: cfg.mountPoint, - Source: cfg.volume, - Type: "volume", - ReadOnly: !cfg.writeable - }; - }); - } - - /** - * Creates container based on provided parameters. - * - * @param containerCfg Image to start container from. - * @returns {Promise} Promise resolving with created container id. - */ - async createContainer( - containerCfg: { - dockerImage: DockerImage, - volumes: DockerAdapterVolumeConfig[], - binds: string[], - ports: any, - envs: string[], - autoRemove: boolean, - maxMem: number, // TODO: Container configuration - command?: string[], - publishAllPorts: boolean, - labels: { [key: string]: string }, - networkMode?: string - } - ): Promise { - containerCfg.ports = { ...containerCfg.ports }; - const config: Dockerode.ContainerCreateOptions = { - Image: containerCfg.dockerImage, - AttachStdin: true, - AttachStdout: true, - AttachStderr: true, - Tty: false, - OpenStdin: true, - StdinOnce: true, - Env: containerCfg.envs, - ExposedPorts: containerCfg.ports.ExposedPorts, - HostConfig: { - Binds: containerCfg.binds, - Mounts: this.translateVolumesConfig(containerCfg.volumes), - AutoRemove: containerCfg.autoRemove || false, - Memory: containerCfg.maxMem, - MemorySwap: 0, - PortBindings: containerCfg.ports.PortBindings, - PublishAllPorts: containerCfg.publishAllPorts || false, - NetworkMode: containerCfg.networkMode - }, - Labels: containerCfg.labels || {}, - }; - - if (containerCfg.command) { - config.Cmd = [...containerCfg.command]; - } - - const { id } = await this.dockerode.createContainer(config); - - return id; - } - - /** - * Start container with provided id. - * - * @param containerId Container id. - * @returns Promise resolving when container has been started. - */ - startContainer(containerId: DockerContainer): Promise { - return this.dockerode.getContainer(containerId).start(); - } - - /** - * Stops container with provided id. - * - * @param containerId Container id. - * @returns Promise which resolves when the container has been stopped. - */ - stopContainer(containerId: DockerContainer): Promise { - return this.dockerode.getContainer(containerId).stop().catch((error: any) => { - this.logger.warn("Failed to stop container"); - - if (error.statusCode === 304) { - this.logger.warn("Container is already stopped"); - return; - } - - throw error; - }); - } - - /** - * Forcefully removes container with provided id. - * - * @param containerId Container id. - * @returns Promise which resolves when container has been removed. - */ - removeContainer(containerId: DockerContainer): Promise { - return this.dockerode.getContainer(containerId).remove(); - } - - /** - * Gets statistics from container with provided id. - * - * @param containerId Container id. - * @returns Promise which resolves with container statistics. - */ - async stats(containerId: DockerContainer): Promise { - return this.dockerode.getContainer(containerId).stats({ stream: false }); - } - - private async isImageInLocalRegistry(name: string): Promise { - return this.dockerode.getImage(name).get().then(() => true, () => false); - } - - private pulledImages: {[key: string]: Promise | undefined } = {}; - - async pullImage(name: string, fetchOnlyIfNotExists = true) { - if (fetchOnlyIfNotExists) { - const start = new Date(); - - this.logger.trace("Checking image", name); - - if (this.pulledImages[name]) { - this.logger.trace("Image already pulled"); - - return this.pulledImages[name]; - } - - if (await this.isImageInLocalRegistry(name)) { - this.logger.trace("Image found in local registry"); - this.pulledImages[name] = Promise.resolve(); - - const seconds = (new Date().getTime() - start.getTime()) / 1000; - - appendFile("timing-log.ndjson", JSON.stringify({ - operation: "checking image", - image: name, - time: seconds, - }) + "\n", () => {}); - - return this.pulledImages[name]; - } - } - - this.pulledImages[name] = (async () => { - const start = new Date(); - - this.logger.trace("Start pulling image", name); - - const pullStream = await this.dockerode.pull(name); - - // Wait for pull to finish - await new Promise(res => this.dockerode.modem.followProgress(pullStream, res)); - - const seconds = (new Date().getTime() - start.getTime()) / 1000; - - appendFile("timing-log.ndjson", JSON.stringify({ - operation: "docker pull", - image: name, - time: seconds, - }) + "\n", () => {}); - - this.logger.trace(`Image pulled in ${seconds}s`); - })(); - - return this.pulledImages[name]; - } - - /** - * Creates docker volume. - * - * @param name Volume name. Optional. If not provided, volume will be named with unique name. - * @returns Volume name. - */ - async createVolume(name: string = ""): Promise { - return this.dockerode.createVolume({ - Name: name, - Labels: { - "org.scramjet.host.is-sequence": "true" - } - }).then((volume: Dockerode.Volume) => { - return volume.name; - }); - } - - /** - * Removes volume with specific name. - * - * @param volumeName Volume name. - * @returns Promise which resolves when volume has been removed. - */ - async removeVolume(volumeName: DockerVolume): Promise { - return this.dockerode.getVolume(volumeName).remove(); - } - - async listVolumes() { - const { Volumes } = await this.dockerode.listVolumes({ - filters: { label: { "org.scramjet.host.is-sequence": true } } - }); - - return Volumes.map(volume => volume.Name); - } - - /** - * Attaches to container streams. - * - * @param container Container id. - * @param opts Attach options. - * @returns Object with container's standard I/O streams. - */ - async attach(container: DockerContainer, opts: any): Promise { - return this.dockerode.getContainer(container).attach(opts); - } - - /** - * Starts container. - * - * @param config Container configuration. - * @returns @see {DockerAdapterRunResponse} - */ - async run(config: DockerAdapterRunConfig): Promise { - const streams: DockerAdapterStreams = { - stdin: new PassThrough(), - stdout: new PassThrough(), - stderr: new PassThrough() - }; - // ------ - const container = await this.createContainer( - { - dockerImage: config.imageName, - volumes: config.volumes || [], - binds: config.binds || [], - ports: config.ports, - envs: config.envs || [], - autoRemove: config.autoRemove || false, - maxMem: (config.maxMem || 64) * 1024 * 1024, - command: config.command, - labels: config.labels || {}, - publishAllPorts: config.publishAllPorts || false, - networkMode: config.networkMode - } - ); - // ------ - const stream = await this.attach(container, { - stream: true, - stdin: true, - stdout: true, - stderr: true, - hijack: true - }); - - stream.on("close", () => { - streams.stdout.emit("end"); - streams.stderr.emit("end"); - }); - - await this.startContainer(container); - - streams.stdin.pipe(stream); - - this.dockerode.getContainer(container) - .modem.demuxStream(stream, streams.stdout, streams.stderr); - - return { - streams: streams, - containerId: container, - wait: async () => this.wait(container, { condition: "not-running" }) - }; - } - - /** - * Waits for container status change. - * - * @param container Container id. - * @param options Condition to be fulfilled. @see {DockerAdapterWaitOptions} - * @returns Container exit code. - */ - async wait(container: DockerContainer, options: DockerAdapterWaitOptions = {}): Promise { - const containerExitResult = await this.dockerode.getContainer(container).wait(options); - - return { statusCode: containerExitResult.StatusCode }; - } - - async listNetworks(): Promise { - // @TODO this - return this.dockerode.listNetworks(); - } - - async inspectNetwork(id: string): Promise { - const network = await this.dockerode.getNetwork(id).inspect(); - - const dockerodeContainers = network.Containers as Record; - - const containers = Object.fromEntries( - Object.entries(dockerodeContainers).map(([containerId, { Name }]) => [containerId, { name: Name }]) - ); - - return { - containers - }; - } - - async connectToNetwork(networkid: string, container: string): Promise { - await this.dockerode.getNetwork(networkid).connect({ Container: container }); - } - - async createNetwork(config: DockerCreateNetworkConfig): Promise { - await this.dockerode.createNetwork({ - Name: config.name, - Driver:config.driver, - Options: config.options - }); - } - - static async isDockerConfigured() { - return isDockerConfigured(); - } -} diff --git a/packages/adapters/src/get-instance-adapter.ts b/packages/adapters/src/get-instance-adapter.ts deleted file mode 100644 index 92444db49..000000000 --- a/packages/adapters/src/get-instance-adapter.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { ProcessInstanceAdapter } from "./process-instance-adapter"; -import { DockerInstanceAdapter } from "./docker-instance-adapter"; -import { ILifeCycleAdapterMain, ILifeCycleAdapterRun, STHConfiguration } from "@scramjet/types"; -import { KubernetesInstanceAdapter } from "./kubernetes-instance-adapter"; - -type InstanceAdapterClass = {new (config: STHConfiguration, id?: string): ILifeCycleAdapterMain & ILifeCycleAdapterRun}; - -const instanceAdapters: Record< -STHConfiguration["runtimeAdapter"], -InstanceAdapterClass -> = { - docker: DockerInstanceAdapter, - process: ProcessInstanceAdapter, - kubernetes: KubernetesInstanceAdapter -}; - -/** - * Provides Instance adapter. - * - * @param runtimeAdapter STH runtime adapter. - * @param config STH config. - * @param id Instance id. - * - * @returns Instance adapter. - */ -export function getInstanceAdapter(runtimeAdapter: STHConfiguration["runtimeAdapter"], config: STHConfiguration, id: string): - ILifeCycleAdapterMain & ILifeCycleAdapterRun { - if (!(runtimeAdapter in instanceAdapters)) { - throw new Error(`Invalid runtimeAdapter ${runtimeAdapter}`); - } - - return new instanceAdapters[runtimeAdapter](config, id); -} diff --git a/packages/adapters/src/get-runner-env.ts b/packages/adapters/src/get-runner-env.ts deleted file mode 100644 index e658ac99e..000000000 --- a/packages/adapters/src/get-runner-env.ts +++ /dev/null @@ -1,40 +0,0 @@ -import path from "path"; -import { RunnerEnvConfig, RunnerEnvironmentVariables } from "./types"; - -/** - * Genrates the required runner env variables - * - * @param conf main parameters - * @param extra any extra parameters - * @returns env vars - */ -export function getRunnerEnvVariables({ - sequencePath, instancesServerPort, instancesServerHost, instanceId, pipesPath, paths = "posix" -}: RunnerEnvConfig, extra: Record = {}): RunnerEnvironmentVariables { - const join = path[paths].join; - - return { - PATH: process.env.PATH, - DEVELOPMENT: process.env.DEVELOPMENT, - PRODUCTION: process.env.PRODUCTION, - SEQUENCE_PATH: sequencePath, - INSTANCES_SERVER_PORT: `${instancesServerPort}`, - INSTANCES_SERVER_HOST: instancesServerHost, - INSTANCE_ID: instanceId, - PIPES_LOCATION: pipesPath, - CRASH_LOG: join(pipesPath, "crash_log"), - ...extra - }; -} - -/** - * Genrates the required runner env variables as Object.entries - * - * @param conf main parameters - * @param extra any extra parameters - * @returns env vars as entries - */ -export function getRunnerEnvEntries(conf: RunnerEnvConfig, extra: Record = {}) { - return Object.entries(getRunnerEnvVariables(conf, extra)); -} - diff --git a/packages/adapters/src/get-sequence-adapter.ts b/packages/adapters/src/get-sequence-adapter.ts deleted file mode 100644 index bbd0351f6..000000000 --- a/packages/adapters/src/get-sequence-adapter.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { ProcessSequenceAdapter } from "./process-sequence-adapter"; -import { DockerSequenceAdapter } from "./docker-sequence-adapter"; -import { ISequenceAdapter, STHConfiguration } from "@scramjet/types"; -import { KubernetesSequenceAdapter } from "./kubernetes-sequence-adapter"; - -type SequenceAdapterClass = {new (config: STHConfiguration): ISequenceAdapter}; - -const sequenceAdapters: Record< - STHConfiguration["runtimeAdapter"], - SequenceAdapterClass -> = { - docker: DockerSequenceAdapter, - process: ProcessSequenceAdapter, - kubernetes: KubernetesSequenceAdapter, -}; - -/** - * Provides Sequence adapter basing on Host configuration. - * - * @param adapter The adapter name - * @param config Host configuration. - * @returns Sequence adapter. - */ -export function getSequenceAdapter(adapter: STHConfiguration["runtimeAdapter"], config: STHConfiguration): ISequenceAdapter { - if (!(adapter in sequenceAdapters)) { - throw new Error(`Invalid runtimeAdapter ${adapter}`); - } - - return new sequenceAdapters[adapter](config); -} diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts deleted file mode 100644 index 48997a8cf..000000000 --- a/packages/adapters/src/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -export * from "./dockerode-docker-helper"; -export * from "./docker-instance-adapter"; -export * from "./docker-sequence-adapter"; -export * from "./initialize-runtime-adapters"; -export * from "./types"; - -export * from "./get-instance-adapter"; -export * from "./get-sequence-adapter"; - -export * from "./docker-networking"; - -export * from "./kubernetes-sequence-adapter"; -export * from "./kubernetes-instance-adapter"; - -export * from "./process-sequence-adapter"; -export * from "./process-instance-adapter"; diff --git a/packages/adapters/src/initialize-runtime-adapters.ts b/packages/adapters/src/initialize-runtime-adapters.ts deleted file mode 100644 index 79a813819..000000000 --- a/packages/adapters/src/initialize-runtime-adapters.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { STHConfiguration } from "@scramjet/types"; -import { setupDockerNetworking } from "./docker-networking"; -import { DockerodeDockerHelper } from "./dockerode-docker-helper"; - -export async function initializeRuntimeAdapters(config: STHConfiguration): Promise { - if (config.runtimeAdapter === "detect") { - if (await DockerodeDockerHelper.isDockerConfigured()) { - config.runtimeAdapter = "docker"; - } else { - config.runtimeAdapter = "process"; - } - } - - if (config.runtimeAdapter === "docker") { - await setupDockerNetworking(new DockerodeDockerHelper()); - } - - if (config.runtimeAdapter === "kubernetes") { - if (!config.kubernetes.sthPodHost) { - throw new Error("Kubernetes pod host url is not set in kubernetes.sthPodHost config."); - } - } - - return config.runtimeAdapter; -} diff --git a/packages/adapters/src/process-instance-adapter.ts b/packages/adapters/src/process-instance-adapter.ts deleted file mode 100644 index fa317b584..000000000 --- a/packages/adapters/src/process-instance-adapter.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { ObjLogger } from "@scramjet/obj-logger"; -import { streamToString } from "@scramjet/utility"; -import { STHConfiguration, - ExitCode, - IComponent, - ILifeCycleAdapterMain, - ILifeCycleAdapterRun, - InstanceConfig, - InstanceLimits, - IObjectLogger, - MonitoringMessageData, - SequenceConfig -} from "@scramjet/types"; -import { ChildProcess, spawn } from "child_process"; - -import path from "path"; -import { getRunnerEnvVariables } from "./get-runner-env"; - -const isTSNode = !!(process as any)[Symbol.for("ts-node.register.instance")]; -const gotPython = "\n _ \n __ _____ _ __ ___ ___| |\n \\ \\ /\\ / / _ \\| '_ \\/ __|_ / |\n \\ V V / (_) | | | \\__ \\/ /|_|\n \\_/\\_/ \\___/|_| |_|___/___(_) 🐍\n"; - -/** - * Adapter for running Instance by Runner executed in separate process. - */ -class ProcessInstanceAdapter implements - ILifeCycleAdapterMain, - ILifeCycleAdapterRun, - IComponent { - logger: IObjectLogger; - sthConfig: STHConfiguration; - - private runnerProcess?: ChildProcess; - private crashLogStreams?: Promise; - private _limits?: InstanceLimits = {}; - - get limits() { return this._limits || {} as InstanceLimits; } - private set limits(value: InstanceLimits) { - this._limits = value; - this.logger.warn("Limits are not yet supported in process runner"); - } - - constructor(config: STHConfiguration) { - this.logger = new ObjLogger(this); - this.sthConfig = config; - } - - async init(): Promise { - // noop - } - async stats(msg: MonitoringMessageData): Promise { - const { runnerProcess } = this; - - if (!runnerProcess) { - // Runner process not initialized yet - return msg; - } - - return { - // @TODO: Provide stats and limits - ...msg, - processId: this.runnerProcess?.pid - }; - } - - getRunnerCmd(config: SequenceConfig) { - const engines = Object.keys(config.engines); - let debugFlags: string[] = []; - - if (engines.length > 1) { - throw new Error("Incorrect config passed to SequenceConfig," + - "'engines' field can't contain more than one element"); - } - - if ("python3" in config.engines) { - this.logger.trace(gotPython); - const runnerPath = path.resolve(__dirname, require.resolve("@scramjet/python-runner")); - - if (this.sthConfig.debug) - debugFlags = ["-m", "pdb", "-c", "continue"]; - - return [ - "/usr/bin/env", - "python3", - ...debugFlags, - path.resolve(__dirname, runnerPath), - "./python-runner-startup.log", - ]; - } - if (this.sthConfig.debug) - debugFlags = ["--inspect-brk=9229"]; - - return [ - isTSNode ? "ts-node" : process.execPath, - ...debugFlags, - path.resolve(__dirname, - process.env.ESBUILD - ? "../../runner/bin/start-runner.js" - : require.resolve("@scramjet/runner") - ) - ]; - } - - getPythonpath(sequenceDir: string) { - // This is for running from source. When the package is built, dependencies - // are installed next to runner.py script (rather than in __pypackages__), - // but that directory is automatically included in PYTHONPATH. - let pythonpath = path.resolve( - __dirname, require.resolve("@scramjet/python-runner"), "../__pypackages__" - ); - - if (process.env.PYTHONPATH) pythonpath += `:${process.env.PYTHONPATH}`; - - pythonpath += `:${sequenceDir}/__pypackages__`; - - return pythonpath; - } - - async run(config: InstanceConfig, instancesServerPort: number, instanceId: string): Promise { - if (config.type !== "process") { - throw new Error("Process instance adapter run with invalid runner config"); - } - - this.logger.info("Instance preparation done"); - - this.logger.trace("Starting Runner", config.id); - - const runnerCommand = this.getRunnerCmd(config); - const sequencePath = path.join( - config.sequenceDir, - config.entrypointPath - ); - const env = getRunnerEnvVariables({ - sequencePath, - instancesServerHost: "127.0.0.1", - instancesServerPort, - instanceId, - pipesPath: "" - }, { - PYTHONPATH: this.getPythonpath(config.sequenceDir), - }); - - this.logger.debug("Spawning Runner process with command", runnerCommand); - this.logger.trace("Runner process environment", env); - - const runnerProcess = spawn(runnerCommand[0], runnerCommand.slice(1), { env }); - - this.crashLogStreams = Promise.all([runnerProcess.stdout, runnerProcess.stderr].map(streamToString)); - - this.logger.trace("Runner process is running", runnerProcess.pid); - - this.runnerProcess = runnerProcess; - - const [statusCode, signal] = await new Promise<[number | null, NodeJS.Signals | null]>( - (res) => runnerProcess.on("exit", (code, sig) => res([code, sig])) - ); - - this.logger.trace("Runner process exited", runnerProcess.pid); - - if (statusCode === null) { - this.logger.warn("Runner was killed by a signal, and didn't return a status code", signal); - - // Probably SIGIKLL - return 137; - } - - if (statusCode > 0) { - this.logger.debug("Process returned non-zero status code", statusCode); - } - - return statusCode; - } - - /** - * Performs cleanup after Runner end. - * Removes fifos used to communication with runner. - */ - async cleanup(): Promise { - //noop - } - - // @ts-ignore - monitorRate(_rps: number): this { - /** ignore */ - } - - /** - * Forcefully stops Runner process. - */ - async remove() { - this.runnerProcess?.kill(); - } - - async getCrashLog(): Promise { - if (!this.crashLogStreams) return []; - - return this.crashLogStreams; - } -} - -export { ProcessInstanceAdapter }; diff --git a/packages/adapters/src/process-sequence-adapter.ts b/packages/adapters/src/process-sequence-adapter.ts deleted file mode 100644 index 02c287480..000000000 --- a/packages/adapters/src/process-sequence-adapter.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { ObjLogger } from "@scramjet/obj-logger"; -import { - ProcessSequenceConfig, - ISequenceAdapter, - STHConfiguration, - SequenceConfig, - IObjectLogger, -} from "@scramjet/types"; -import { Readable } from "stream"; -import { createReadStream } from "fs"; -import fs from "fs/promises"; -import path from "path"; -import { exec } from "child_process"; -import { isDefined, readStreamedJSON } from "@scramjet/utility"; -import { sequencePackageJSONDecoder } from "./validate-sequence-package-json"; -import { SequenceAdapterError } from "@scramjet/model"; -import { detectLanguage } from "./utils"; - -/** - * Returns existing Sequence configuration. - * - * @param {string} sequencesRoot Folder where sequences are located. - * @param {string} id Sequence Id. - * @returns {ProcessSequenceConfig} Sequence configuration. - */ -// eslint-disable-next-line complexity -async function getRunnerConfigForStoredSequence(sequencesRoot: string, id: string): Promise { - const sequenceDir = path.join(sequencesRoot, id); - const packageJsonPath = path.join(sequenceDir, "package.json"); - const packageJson = await readStreamedJSON(createReadStream(packageJsonPath)); - - const validPackageJson = await sequencePackageJSONDecoder.decodeToPromise(packageJson); - const engines = validPackageJson.engines ? { ...validPackageJson.engines } : {}; - - return { - type: "process", - engines, - entrypointPath: validPackageJson.main, - version: validPackageJson.version ?? "", - name: validPackageJson.name ?? "", - id, - sequenceDir, - description: validPackageJson.description, - author: validPackageJson.author, - keywords: validPackageJson.keywords, - args: validPackageJson.args, - repository: validPackageJson.repository, - language: detectLanguage(validPackageJson) - }; -} - -/** - * Adapter for preparing Sequence to be run in process. - */ -class ProcessSequenceAdapter implements ISequenceAdapter { - logger: IObjectLogger; - - name = "ProcessSequenceAdapter"; - config: NonNullable; - - constructor(config: STHConfiguration) { - this.logger = new ObjLogger(this); - this.config = config.adapters["@scramjet/adapter-process"]!; - } - - /** - * Initializes adapter. - * - * @returns {Promise} Promise resolving after initialization. - */ - async init(): Promise { - await fs.access(this.config.sequencesRoot) - .catch(() => fs.mkdir(this.config.sequencesRoot)); - - this.logger.info("Proces adapter initialized with options", { - "sequence root": this.config.sequencesRoot - }); - } - - /** - * Finds existing sequences. - * - * @returns {Promise} Promise resolving to array of identified sequences. - */ - async list(): Promise { - const storedSequencesIds = await fs.readdir(this.config.sequencesRoot); - const sequencesConfigs = (await Promise.all( - storedSequencesIds - .map((id) => getRunnerConfigForStoredSequence(this.config.sequencesRoot, id)) - .map((configPromised) => configPromised.catch(() => null)) - )) - .filter(isDefined); - - this.logger.debug(`Found ${sequencesConfigs.length} stored sequences`); - - return sequencesConfigs; - } - - /** - * Unpacks and identifies sequence. - * - * @param {Readable} stream Stream with packed sequence. - * @param {string} id Sequence Id. - * @param {boolean} override Removes previous sequence - * @returns {Promise} Promise resolving to identified sequence configuration. - */ - async identify(stream: Readable, id: string, override = false): Promise { - const sequenceDir = path.join(this.config.sequencesRoot, id); - - if (override) { - await fs.rm(sequenceDir, { recursive: true, force: true }); - } - - await fs.mkdir(sequenceDir, { recursive: true }); - - const uncompressingProc = exec(`tar zxf - -C ${sequenceDir} >/dev/null 2>&1 || echo >&2 '{"error":"Invalid pkg tar.gz archive"}' && exit 1`); - - stream.pipe(uncompressingProc.stdin!); - - const stderrChunks: string[] = []; - - uncompressingProc.stderr!.on("data", (chunk: Buffer) => { - stderrChunks.push(chunk.toString()); - }); - - await new Promise(res => uncompressingProc.on("close", res)); - - const stderrOutput = stderrChunks.join(""); - - if (stderrOutput) { - let preRunnenrError; - - try { - preRunnenrError = JSON.parse(stderrOutput); - this.logger.error("Unpacking sequence failed", stderrOutput); - } catch (e) { - throw new SequenceAdapterError("PRERUNNER_ERROR", `Error parsing ${stderrOutput}`); - } - - throw new SequenceAdapterError("PRERUNNER_ERROR", preRunnenrError.error); - } - - this.logger.debug("Unpacking sequence succeeded", stderrOutput); - - return getRunnerConfigForStoredSequence(this.config.sequencesRoot, id); - } - - /** - * Removes directory used to store sequence. - * - * @param {SequenceConfig} config Sequence configuration. - * @returns {Promise} Promise resolving after directory deletion. - */ - async remove(config: SequenceConfig) { - if (config.type !== "process") { - throw new Error(`Incorrect SequenceConfig passed to ProcessSequenceAdapter: ${config.type}`); - } - - const sequenceDir = path.join(this.config.sequencesRoot, config.id); - - return fs.rm(sequenceDir, { recursive: true }); - } -} - -export { ProcessSequenceAdapter }; diff --git a/packages/adapters/src/types.ts b/packages/adapters/src/types.ts deleted file mode 100644 index b5ec64fbc..000000000 --- a/packages/adapters/src/types.ts +++ /dev/null @@ -1,330 +0,0 @@ -import { ExitCode, InstanceId, IObjectLogger } from "@scramjet/types"; -import { ContainerStats, NetworkInspectInfo } from "dockerode"; -import { PathLike } from "fs"; -import { Stream, Writable } from "stream"; - -/** - * Docker image. - * - * @typedef {string} DockerImage - */ -export type DockerImage = string; - -/** - * Docker container. - * - * @typedef {string} DockerContainer - */ -export type DockerContainer = string; - -/** - * Docker volume. - * - * @typedef {string} DockerVolume - */ -export type DockerVolume = string; - -/** - * Volume mounting configuration. - * - * @typedef {object} DockerAdapterVolumeConfig - */ -export type DockerAdapterVolumeConfig = { - /** - * @property {string} mountPoint Mount point. - * */ - mountPoint: string; - - /** - * @property { boolean } writeable Mount mode. Container can write to the volume if set to true. - */ - writeable: boolean; -} & ( - { - /** - * @property {DockerVolume} volume Volume. - */ - volume: DockerVolume; - } | { - /** - * @property {string} bind A bind mount. - */ - bind: string; - } -); - -export type DockerAdapterRunPortsConfig = { - ExposedPorts: any, - PortBindings: any -} - -export type DockerNetwork = { containers: Record } - -export type DockerCreateNetworkConfig = { name: string, driver: string, options: Record } - -/** - * Configuration used to run command in container. - * - * @typedef {object} DockerAdapterRunConfig - */ -export type DockerAdapterRunConfig = { - /** - * @property {string} imageName Image name. - */ - imageName: string; - - /** - * Command with optional parameters. - * - * @property {string[]} command Command to be executed. - */ - command?: string[]; - - /** - * @property {DockerAdapterVolumeConfig[]} volumes Volumes configuration. - */ - volumes?: DockerAdapterVolumeConfig[], - - /** - * @property {string[]} binds Directories mount configuration. - */ - binds?: string[], - - /** - * @property {DockerAdapterRunPortsConfig} ports Docker ports configuration - */ - ports?: DockerAdapterRunPortsConfig - - /** - * @property {string[]} envs A list of environment variables - * to set inside the container in the form ```["VAR=value", ...]``` - */ - envs?: string[], - - /** - * @property {boolean} autoRemove If true container will be removed after container's process exit. - */ - autoRemove?: boolean, - - /** - * @property {number} maxMem Container memory limit (bytes). - */ - maxMem?: number, - - publishAllPorts?: boolean, - - labels?: { - [key: string]: string - }, - - networkMode?: string, -}; - -/** - * Standard streams connected with container. - */ -export type DockerAdapterStreams = { - /** - * @type {Writable} - */ - stdin: Writable, - - /** - * @type {Stream} - */ - stdout: Stream, - - /** - * @type {Stream} - */ - stderr: Stream -}; - -export type ExitData = { - statusCode: ExitCode -} - -export type DockerAdapterResources = { - containerId?: DockerContainer; - volumeId?: DockerVolume; - fifosDir?: PathLike; - ports?: DockerAdapterRunPortsConfig; -} - -export type DockerAdapterWaitOptions = { - condition?: "not-running" | "next-exit" | "removed" -} - -/** - * Result of running command in container. - */ -export type DockerAdapterRunResponse = { - /** - * @type {DockerAdapterStreams} Set of standard streams. - */ - streams: DockerAdapterStreams, - - /** - * @type {Function} Function which return promise resolving when container status changed. - * Used to wait for container end. - */ - wait: Function - - /** - * @type {DockerContainer} Docker container. - */ - containerId: DockerContainer -}; -export interface IDockerHelper { - logger: IObjectLogger; - - /** - * Converts pairs of mount path and volume name to DockerHelper specific volume configuration. - * - * @param {DockerAdapterVolumeConfig} volumeConfigs[] Volume configuration objects. - * - * @returns {any} DockerHelper volume configuration. - */ - translateVolumesConfig: (volumeConfigs: DockerAdapterVolumeConfig[]) => any; - - /** - * Creates Docker container from provided image with attached volumes and local directories. - * - * @param {DockerImage} dockerImage Docker image name. - * @param {DockerAdapterVolumeConfig[]} volumes Volumes to be mounted to container. - * @param {string[]} binds Directories to be mounted. - * @param {string[]} envs Environment variables. - * @param {boolean} autoRemove If true, container will be removed when finished. - * - * @returns {Promise} Created container. - */ - createContainer: ( - containerCfg: { - dockerImage: DockerImage, - volumes: DockerAdapterVolumeConfig[], - binds: string[], - ports: any, - envs: string[], - autoRemove: boolean, - maxMem: number, - publishAllPorts: boolean, - labels: { - [key: string]: string - }, - networkMode?: string - } - ) => Promise; - - /** - * Starts container. - * - * @param {DockerContainer} containerId Container to be started. - * - * @returns {Promise} - */ - startContainer: (containerId: DockerContainer) => Promise; - - /** - * Stops container. - * - * @param {DockerContainer} containerId Container id to be stopped. - * - * @returns {Promise} - */ - stopContainer: (containerId: DockerContainer) => Promise; - - stats: (containerId: DockerContainer) => Promise; - /** - * Removes container. - * - * @param {DockerContainer} containerId Container id. - * - * @returns {Promise} - */ - removeContainer: (containerId: DockerContainer) => Promise; - - /** - * Lists existing volumes - * - * @returns {Promise} List of existing volumes - */ - listVolumes: () => Promise; - - /** - * Creates volume. - * - * @param {string} name Volume name. - * - * @returns {Promise} Created volume. - */ - createVolume: (name?: string) => Promise; - - /** - * Removes volume. - * - * @param {DockerVolume} Volume. - * - * @returns {Promise} - */ - removeVolume: (volumeId: DockerVolume) => Promise; - - /** - * Executes command in container. - * - * @param {DockerAdapterRunConfig} config Execution configuration. - * - * @returns {Promise} - */ - run: (config: DockerAdapterRunConfig) => Promise; - - /** - * Waits until container exits - * - * @param {DockerContainer} container - * - * @returns {Promise} - */ - wait(container: DockerContainer, options?: DockerAdapterWaitOptions): Promise; - - /** - * Fetches the image from repo - * - * @param name the name of the image, eg. ubuntu:latest - * @param fetchOnlyIfNotExists fetch only if not exists (defaults to true) - */ - pullImage(name: string, fetchOnlyIfNotExists?: boolean): Promise - - listNetworks(): Promise - - inspectNetwork(id: string): Promise - - connectToNetwork(networkid: string, container: string): Promise - - createNetwork(config: DockerCreateNetworkConfig): Promise -} - -export type InstanceAdapterOptions = { - exitDelay: number; -} - -export type RunnerEnvConfig = { - paths?: "posix" | "win32" - sequencePath: string; - pipesPath: string; - instancesServerPort: number; - instancesServerHost: string; - instanceId: InstanceId; -} - -export type RunnerEnvironmentVariables = Partial<{ - PATH: string; - DEVELOPMENT: string; - PRODUCTION: string; - SEQUENCE_PATH: string; - INSTANCES_SERVER_PORT: string; - INSTANCES_SERVER_HOST: string; - INSTANCE_ID: string; - PIPES_LOCATION: string; - CRASH_LOG: string; - [key: string]: string; -}>; diff --git a/packages/adapters/src/utils.ts b/packages/adapters/src/utils.ts deleted file mode 100644 index ce2e872cd..000000000 --- a/packages/adapters/src/utils.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Determines Sequence language by checking file extension in `main` field. - * When failed, checks `engines` field for `node`. - * Returns "unknown" when language can't be determined with this methods. - * - * @param {any} packageJson package.json contents - * @returns {string} Detected language or "unknown" - */ -export const detectLanguage = (packageJson: {[key: string]: any}) => { - if (packageJson.engines) { - if ("python3" in packageJson.engines) return "py"; - if ("node" in packageJson.engines) return "js"; - } - - return (packageJson.main?.match(/(?:\.)([^.\\/:*?"<>|\r\n]+$)/) || { 1: undefined })[1] || "unknown"; -}; diff --git a/packages/adapters/src/validate-sequence-package-json.ts b/packages/adapters/src/validate-sequence-package-json.ts deleted file mode 100644 index fca2df88b..000000000 --- a/packages/adapters/src/validate-sequence-package-json.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { - PortConfig, - SequencePackageJSON, - SequencePackageJSONScramjetConfig, - SequencePackageJSONScramjetSection -} from "@scramjet/types"; - -import { Err, JsonDecoder, Ok } from "ts.data.json"; - -const enginesDecoder = JsonDecoder.dictionary(JsonDecoder.string, "EnginesDecoder"); - -const portDecoder = new JsonDecoder.Decoder((val) => { - const isValid = typeof val === "string" && (/^\d{3,5}\/(tcp|udp)$/).test(val); - - return isValid ? new Ok(val as PortConfig) : new Err("Invalid port format"); -}); - -const configDecoder = JsonDecoder.object({ - ports: JsonDecoder.optional(JsonDecoder.array(portDecoder, "PortsDecoder")) -}, "ConfigDecoder"); - -const scramjetDecoder = JsonDecoder.object({ - config: JsonDecoder.optional(configDecoder) -}, "ScramjetSectionDecoder"); - -type RepositoryObject = { - type: string; - url: string; - directory?: string; -} - -const repositoryDecoder = JsonDecoder.oneOf([ - JsonDecoder.string, - JsonDecoder.object({ - type: JsonDecoder.string, - url: JsonDecoder.string, - directory: JsonDecoder.optional(JsonDecoder.string) - }, "repositoryObjectDecoder") -], "repositoryDecoder"); - -export const sequencePackageJSONDecoder = JsonDecoder.object({ - name: JsonDecoder.optional(JsonDecoder.string), - version: JsonDecoder.optional(JsonDecoder.string), - main: JsonDecoder.string, - engines: JsonDecoder.optional(enginesDecoder), - scramjet: JsonDecoder.optional(scramjetDecoder), - description: JsonDecoder.optional(JsonDecoder.string), - author: JsonDecoder.optional(JsonDecoder.string), - keywords: JsonDecoder.optional(JsonDecoder.array(JsonDecoder.string, "keywordsDecoder")), - repository: JsonDecoder.optional(repositoryDecoder), - args: JsonDecoder.optional(JsonDecoder.array(JsonDecoder.succeed, "argsDecoder")), -}, "SequencePackageJSON"); diff --git a/packages/host/src/lib/adapter-manager.ts b/packages/host/src/lib/adapter-manager.ts index 2392c39b3..47fd2505b 100644 --- a/packages/host/src/lib/adapter-manager.ts +++ b/packages/host/src/lib/adapter-manager.ts @@ -62,7 +62,9 @@ export class AdapterManager { * @param _prerequesities Prerequesities * @returns IRuntimeAdapter */ - getAvailableAdapter(_prerequesities: { requirements?: InstanceRequirements, adapterName?: string } = {}): IRuntimeAdapter | undefined { + getAvailableAdapter(_prerequesities: { + requirements?: InstanceRequirements, adapterName?: string + } = {}): IRuntimeAdapter | undefined { if (_prerequesities.adapterName) { return this.getAdapterByName(_prerequesities.adapterName); } diff --git a/packages/host/src/lib/host.ts b/packages/host/src/lib/host.ts index 80db3591a..041568988 100644 --- a/packages/host/src/lib/host.ts +++ b/packages/host/src/lib/host.ts @@ -995,7 +995,9 @@ export class Host implements IComponent { this.logger.debug("CSIC start payload", payload); - const csic = new CSIController(id, sequence, payload, communicationHandler, this.config, this.instanceProxy, instanceAdapter); + const csic = new CSIController( + id, sequence, payload, communicationHandler, this.config, this.instanceProxy, instanceAdapter + ); csic.logger.pipe(this.logger, { end: false }); communicationHandler.logger.pipe(this.logger, { end: false }); diff --git a/yarn.lock b/yarn.lock index a31220052..519143e09 100644 --- a/yarn.lock +++ b/yarn.lock @@ -814,6 +814,26 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@scramjet/adapters@^0.34.4": + version "0.34.4" + resolved "https://registry.yarnpkg.com/@scramjet/adapters/-/adapters-0.34.4.tgz#7de9d89c9c8f2730b2c8e491442cf4fea72bc633" + integrity sha512-gl42dOUNRaPriv+AA0dXm4/2co7x3ZgnmqatYn7gx1fG8DsXWaIt0DYCgQ9Gj3sHP0B6qFdD9hNiDkBoBJZZUQ== + dependencies: + "@kubernetes/client-node" "^0.17.1" + "@scramjet/model" "^0.34.4" + "@scramjet/obj-logger" "^0.34.4" + "@scramjet/pre-runner" "^0.34.4" + "@scramjet/python-runner" "^0.34.4" + "@scramjet/runner" "^0.34.4" + "@scramjet/sth-config" "^0.34.4" + "@scramjet/symbols" "^0.34.4" + "@scramjet/utility" "^0.34.4" + dockerode "^3.3.4" + scramjet "^4.36.9" + shell-escape "^0.2.0" + systeminformation "^5.12.7" + ts.data.json "^2.2.0" + "@sindresorhus/is@^0.14.0": version "0.14.0" resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" @@ -903,22 +923,6 @@ resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.2.tgz#f65d3d6389e01eeb458bd54dc8f52b95a9463bc8" integrity sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w== -"@types/docker-modem@*": - version "3.0.2" - resolved "https://registry.yarnpkg.com/@types/docker-modem/-/docker-modem-3.0.2.tgz#c49c902e17364fc724e050db5c1d2b298c6379d4" - integrity sha512-qC7prjoEYR2QEe6SmCVfB1x3rfcQtUr1n4x89+3e0wSTMQ/KYCyf+/RAA9n2tllkkNc6//JMUZePdFRiGIWfaQ== - dependencies: - "@types/node" "*" - "@types/ssh2" "*" - -"@types/dockerode@<=3.3.3": - version "3.3.3" - resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-3.3.3.tgz#993de5d8bd3ecb0b0a9277cccc189e9a74e9fbd1" - integrity sha512-DBFTE/wiZKodQhDrdkzLVT3K0X1JO/1fTxPaTQN0jiPNdTOFoPt8LYnuYEzalPgVpZbGEeuJTls7qn6jMvuydg== - dependencies: - "@types/docker-modem" "*" - "@types/node" "*" - "@types/find-package-json@^1.2.3": version "1.2.3" resolved "https://registry.yarnpkg.com/@types/find-package-json/-/find-package-json-1.2.3.tgz#45482a72eb8a21fcc330adff774fb9c7971e6ced" @@ -959,7 +963,7 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node@*", "@types/node@>=13.7.0", "@types/node@^18.11.18": +"@types/node@*", "@types/node@>=13.7.0": version "18.14.2" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.14.2.tgz#c076ed1d7b6095078ad3cf21dfeea951842778b1" integrity sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA== @@ -1011,13 +1015,6 @@ resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.2.tgz#bf2e02a3dbd4aecaf95942ecd99b7402e03fad5e" integrity sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA== -"@types/ssh2@*": - version "1.11.7" - resolved "https://registry.yarnpkg.com/@types/ssh2/-/ssh2-1.11.7.tgz#41b7477787a7fcb07b5d16cf907ed545ff0c1017" - integrity sha512-MaVSlZOiekRUHnxL2NAmDkcU3+bTwz+ZRktLygjQCnxhLjDzN+XnsG6JUdoocmfxatMiqFo/6eb48uVqFaxBsg== - dependencies: - "@types/node" "^18.11.18" - "@types/tar@^6.1.3": version "6.1.4" resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.4.tgz#cf8497e1ebdc09212fd51625cd2eb5ca18365ad1"