Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Yagna client module #558

Merged
merged 23 commits into from
Aug 29, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
c5954d0
JST-61: Yagna Connection draft
mgordel Aug 10, 2023
b8e5b45
Merge remote-tracking branch 'origin/master' into bugfix/JST-61/yagna…
mgordel Aug 11, 2023
9eb1b7c
JST-61: Added global yagna object
mgordel Aug 11, 2023
95a8ca7
Merge remote-tracking branch 'origin/master' into bugfix/JST-61/yagna…
mgordel Aug 11, 2023
ffc5151
fix(executor): added global YagnaApi module
mgordel Aug 16, 2023
df5d0ce
refactor(executor): added global YagnaApi to TaskExecutor (JST-61)
mgordel Aug 16, 2023
b64b757
Merge remote-tracking branch 'origin/master' into bugfix/JST-61/yagna…
mgordel Aug 16, 2023
e033c61
Merge remote-tracking branch 'origin/master' into bugfix/JST-61/yagna…
mgordel Aug 22, 2023
a38deae
test: fixed unit tests
mgordel Aug 22, 2023
c2a87dd
refactor: fixed unit tests for yagna api module
mgordel Aug 22, 2023
2b31ba3
refactor: fixed unit tests for yagna api module
mgordel Aug 23, 2023
0a547e8
refactor: fixed typedoc params
mgordel Aug 23, 2023
3ae04c8
fix: fixed rollup closing bug
mgordel Aug 23, 2023
c3bd003
Merge remote-tracking branch 'origin/beta' into bugfix/JST-61/yagna-c…
mgordel Aug 23, 2023
c3f599b
fix: removed a workaround for rollup issue which is breaking the pack…
grisha87 Aug 23, 2023
22d832d
Merge pull request #559 from golemfactory/bugfix/JST-287/broken-npm-p…
grisha87 Aug 23, 2023
23fd5f0
Merge branch 'master' into beta
grisha87 Aug 23, 2023
39daae0
refactor: fixed unit tests for yagna module
mgordel Aug 23, 2023
23a585e
build: increased goth setup timeout
mgordel Aug 23, 2023
9463de6
test: removed apikey from process env in tests
mgordel Aug 23, 2023
bacf37a
test: removed apikey from process env in tests
mgordel Aug 23, 2023
2ed0392
Merge remote-tracking branch 'origin/beta' into bugfix/JST-61/yagna-c…
mgordel Aug 24, 2023
c98c184
refactor: fixed tagna network api config
mgordel Aug 24, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions rollup.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,6 @@ export default [
json(), // Required because one our dependencies (bottleneck) loads its own 'version.json'
typescript({ tsconfig: "./tsconfig.json" }),
terser({ keep_classnames: true }),
{
// Temporary workaround https://github.com/rollup/rollup/issues/4213
closeBundle() {
if (!process.env.ROLLUP_WATCH) {
setTimeout(() => process.exit(0));
}
},
name: "force-close",
},
],
},
// NodeJS
Expand Down
39 changes: 21 additions & 18 deletions src/activity/activity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import sleep from "../utils/sleep";
import { ActivityFactory } from "./factory";
import { ActivityConfig } from "./config";
import { Events } from "../events";
import { YagnaApi } from "../utils/yagna/yagna";

/**
* @hidden
Expand All @@ -27,12 +28,6 @@ export interface ExeScriptRequest {
}

export interface ActivityOptions {
yagnaOptions?: {
/** Yagna Api Key */
apiKey?: string;
/** Yagna base path to Activity REST Api */
basePath?: string;
};
/** timeout for sending and creating batch */
activityRequestTimeout?: number;
/** timeout for executing batch */
Expand All @@ -58,12 +53,14 @@ export class Activity {
/**
* @param id activity ID
* @param agreementId agreement ID
* @param yagnaApi - {@link YagnaApi}
* @param options - {@link ActivityOptions}
* @hidden
*/
constructor(
public readonly id,
public readonly agreementId,
protected readonly yagnaApi: YagnaApi,
protected readonly options: ActivityConfig,
) {
this.logger = options?.logger;
Expand All @@ -73,12 +70,18 @@ export class Activity {
* Create activity for given agreement ID
*
* @param agreementId
* @param yagnaApi
* @param options - {@link ActivityOptions}
* @param secure - defines if activity will be secure type
* @return Activity
*/
static async create(agreementId: string, options?: ActivityOptions, secure = false): Promise<Activity> {
const factory = new ActivityFactory(agreementId, options);
static async create(
agreementId: string,
yagnaApi: YagnaApi,
options?: ActivityOptions,
secure = false,
): Promise<Activity> {
const factory = new ActivityFactory(agreementId, yagnaApi, options);
return factory.create(secure);
}

Expand Down Expand Up @@ -128,7 +131,7 @@ export class Activity {
*/
public async getState(): Promise<ActivityStateEnum> {
try {
const { data } = await this.options.api.state.getActivityState(this.id);
const { data } = await this.yagnaApi.activity.state.getActivityState(this.id);
const state = data.state[0];
if (this.currentState !== ActivityStateEnum[state]) {
this.options.eventTarget?.dispatchEvent(
Expand All @@ -144,14 +147,14 @@ export class Activity {
}

protected async send(script: ExeScriptRequest): Promise<string> {
const { data: batchId } = await this.options.api.control.exec(this.id, script, {
const { data: batchId } = await this.yagnaApi.activity.control.exec(this.id, script, {
timeout: this.options.activityRequestTimeout,
});
return batchId;
}

private async end() {
await this.options.api.control
await this.yagnaApi.activity.control
.destroyActivity(this.id, this.options.activityRequestTimeout / 1000, {
timeout: this.options.activityRequestTimeout + 1000,
})
Expand All @@ -161,7 +164,6 @@ export class Activity {
);
});
this.options.eventTarget?.dispatchEvent(new Events.ActivityDestroyed(this));
this.options.httpAgent.destroy?.();
this.logger?.debug(`Activity ${this.id} destroyed`);
}

Expand All @@ -172,7 +174,8 @@ export class Activity {
const maxRetries = 5;
const { id: activityId, agreementId } = this;
const isRunning = () => this.isRunning;
const { activityExecuteTimeout, api, eventTarget } = this.options;
const { activityExecuteTimeout, eventTarget } = this.options;
const api = this.yagnaApi.activity;
const handleError = this.handleError.bind(this);
return new Readable({
objectMode: true,
Expand Down Expand Up @@ -222,8 +225,8 @@ export class Activity {
}

private async streamingBatch(batchId, batchSize, startTime, timeout): Promise<Readable> {
const basePath = this.options?.yagnaOptions?.basePath || this.options.api.control["configuration"]?.basePath;
const apiKey = this.options?.yagnaOptions?.apiKey || this.options.api.control["configuration"]?.apiKey;
const basePath = this.yagnaApi.yagnaOptions.basePath;
const apiKey = this.yagnaApi.yagnaOptions.apiKey;
const eventSource = new EventSource(`${basePath}/activity/${this.id}/exec/${batchId}`, {
headers: {
Accept: "text/event-stream",
Expand Down Expand Up @@ -279,13 +282,13 @@ export class Activity {
throw error;
}
++retryCount;
const failMsg = "There was an error retrieving activity results. ";
const failMsg = "There was an error retrieving activity results.";
const errorMsg = error?.response?.data?.message || error?.message || error;
if (retryCount < maxRetries) {
this.logger?.debug(`${failMsg} Retrying in ${this.options.activityExeBatchResultsFetchInterval}.`);
return retryCount;
} else {
this.logger?.error(`${failMsg} Giving up after ${retryCount} attempts. ${errorMsg}`);
this.logger?.warn(`${failMsg} Giving up after ${retryCount} attempts. ${errorMsg}`);
}
throw new Error(`Command #${cmdIndex || 0} getExecBatchResults error: ${errorMsg}`);
}
Expand Down Expand Up @@ -317,7 +320,7 @@ export class Activity {

private async isTerminated(): Promise<{ terminated: boolean; reason?: string; errorMessage?: string }> {
try {
const { data } = await this.options.api.state.getActivityState(this.id);
const { data } = await this.yagnaApi.activity.state.getActivityState(this.id);
const state = ActivityStateEnum[data?.state?.[0]];
return {
terminated: state === ActivityStateEnum.Terminated,
Expand Down
24 changes: 1 addition & 23 deletions src/activity/config.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
import { ActivityOptions } from "./activity";
import { yaActivity } from "ya-ts-client";
import { RequestorControlApi, RequestorStateApi } from "ya-ts-client/dist/ya-activity/api";
import { EnvUtils, Logger } from "../utils";
import { YagnaOptions } from "../executor";
import { Agent } from "http";
import { Logger } from "../utils";

const DEFAULTS = {
activityRequestTimeout: 10000,
Expand All @@ -15,36 +11,18 @@ const DEFAULTS = {
* @internal
*/
export class ActivityConfig {
public readonly api: { control: RequestorControlApi; state: RequestorStateApi };
public readonly activityRequestTimeout: number;
public readonly activityExecuteTimeout: number;
public readonly activityExeBatchResultsFetchInterval: number;
public readonly logger?: Logger;
public readonly eventTarget?: EventTarget;
public readonly yagnaOptions: YagnaOptions;
public readonly httpAgent: Agent;

constructor(options?: ActivityOptions) {
const apiKey = options?.yagnaOptions?.apiKey || EnvUtils.getYagnaAppKey();
if (!apiKey) throw new Error("Api key not defined");
const basePath = options?.yagnaOptions?.basePath || EnvUtils.getYagnaApiUrl();
this.httpAgent = new Agent({ keepAlive: true });
const apiConfig = new yaActivity.Configuration({
apiKey,
basePath: `${basePath}/activity-api/v1`,
accessToken: apiKey,
baseOptions: { httpAgent: this.httpAgent },
});
this.api = {
control: new RequestorControlApi(apiConfig),
state: new RequestorStateApi(apiConfig),
};
this.activityRequestTimeout = options?.activityRequestTimeout || DEFAULTS.activityRequestTimeout;
this.activityExecuteTimeout = options?.activityExecuteTimeout || DEFAULTS.activityExecuteTimeout;
this.activityExeBatchResultsFetchInterval =
options?.activityExeBatchResultsFetchInterval || DEFAULTS.activityExeBatchResultsFetchInterval;
this.logger = options?.logger;
this.yagnaOptions = { apiKey, basePath };
this.eventTarget = options?.eventTarget;
}
}
13 changes: 8 additions & 5 deletions src/activity/factory.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { Activity, ActivityOptions } from "./activity";
import { ActivityConfig } from "./config";
import { Events } from "../events";
import { YagnaApi } from "../utils/yagna/yagna";

/**
* Activity Factory
Expand All @@ -14,10 +15,12 @@ export class ActivityFactory {
* Creating ActivityFactory
*
* @param agreementId
* @param yagnaApi - {@link YagnaApi}
* @param options - {@link ActivityOptions}
*/
constructor(
private readonly agreementId: string,
private readonly yagnaApi: YagnaApi,
options?: ActivityOptions,
) {
this.options = new ActivityConfig(options);
Expand All @@ -35,19 +38,19 @@ export class ActivityFactory {
if (secure) {
throw new Error("Not implemented");
}
return this.createActivity(this.agreementId, this.options);
return this.createActivity();
} catch (error) {
const msg = `Unable to create activity: ${error?.response?.data?.message || error}`;
this.options.logger?.error(msg);
throw new Error(msg);
}
}

private async createActivity(agreementId: string, options: ActivityConfig): Promise<Activity> {
const { data } = await this.options.api.control.createActivity({ agreementId });
private async createActivity(): Promise<Activity> {
const { data } = await this.yagnaApi.activity.control.createActivity({ agreementId: this.agreementId });
const id = typeof data == "string" ? data : data.activityId;
this.options.logger?.debug(`Activity ${id} created`);
this.options.eventTarget?.dispatchEvent(new Events.ActivityCreated({ id, agreementId }));
return new Activity(id, agreementId, options);
this.options.eventTarget?.dispatchEvent(new Events.ActivityCreated({ id, agreementId: this.agreementId }));
return new Activity(id, this.agreementId, this.yagnaApi, this.options);
}
}
20 changes: 12 additions & 8 deletions src/agreement/agreement.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { YagnaOptions } from "../executor";
import { AgreementFactory } from "./factory";
import { AgreementConfig } from "./config";
import { Events } from "../events";
import { YagnaApi } from "../utils/yagna/yagna";

/**
* @hidden
Expand Down Expand Up @@ -52,12 +53,14 @@ export class Agreement {
/**
* @param id - agreement ID
* @param provider - {@link ProviderInfo}
* @param yagnaApi - {@link YagnaApi}
* @param options - {@link AgreementConfig}
* @hidden
*/
constructor(
public readonly id,
public readonly provider: ProviderInfo,
private readonly yagnaApi: YagnaApi,
private readonly options: AgreementConfig,
) {
this.logger = options.logger;
Expand All @@ -66,19 +69,22 @@ export class Agreement {
/**
* Create agreement for given proposal ID
* @param proposalId - proposal ID
* @param yagnaApi
* @param agreementOptions - {@link AgreementOptions}
* @return Agreement
*/
static async create(proposalId: string, agreementOptions?: AgreementOptions): Promise<Agreement> {
const factory = new AgreementFactory(agreementOptions);
static async create(proposalId: string, yagnaApi: YagnaApi, agreementOptions?: AgreementOptions): Promise<Agreement> {
const factory = new AgreementFactory(yagnaApi, agreementOptions);
return factory.create(proposalId);
}

/**
* Refresh agreement details
*/
async refreshDetails() {
const { data } = await this.options.api.getAgreement(this.id, { timeout: this.options.agreementRequestTimeout });
const { data } = await this.yagnaApi.market.getAgreement(this.id, {
timeout: this.options.agreementRequestTimeout,
});
this.agreementData = data;
}

Expand All @@ -98,8 +104,8 @@ export class Agreement {
*/
async confirm() {
try {
await this.options.api.confirmAgreement(this.id);
await this.options.api.waitForApproval(this.id, this.options.agreementWaitingForApprovalTimeout);
await this.yagnaApi.market.confirmAgreement(this.id);
await this.yagnaApi.market.waitForApproval(this.id, this.options.agreementWaitingForApprovalTimeout);
this.logger?.debug(`Agreement ${this.id} approved`);
this.options.eventTarget?.dispatchEvent(
new Events.AgreementConfirmed({ id: this.id, providerId: this.provider.id }),
Expand Down Expand Up @@ -133,7 +139,7 @@ export class Agreement {
if ((await this.getState()) !== AgreementStateEnum.Terminated)
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore TODO: API binding BUG with reason type
await this.options.api.terminateAgreement(this.id, reason, {
await this.yagnaApi.market.terminateAgreement(this.id, reason, {
timeout: this.options.agreementRequestTimeout,
});
this.options.eventTarget?.dispatchEvent(
Expand All @@ -144,8 +150,6 @@ export class Agreement {
throw new Error(
`Unable to terminate agreement ${this.id}. ${error.response?.data?.message || error.response?.data || error}`,
);
} finally {
this.options.httpAgent.destroy?.();
}
}
}
18 changes: 1 addition & 17 deletions src/agreement/config.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
import { AgreementOptions } from "./agreement";
import { AgreementSelector, AgreementServiceOptions } from "./service";
import { RequestorApi } from "ya-ts-client/dist/ya-market/api";
import { Configuration } from "ya-ts-client/dist/ya-market";
import { EnvUtils, Logger } from "../utils";
import { Logger } from "../utils";
import { randomAgreementSelector } from "./strategy";
import { Agent } from "http";

const DEFAULTS = {
agreementRequestTimeout: 30000,
Expand All @@ -18,23 +15,10 @@ const DEFAULTS = {
export class AgreementConfig {
readonly agreementRequestTimeout: number;
readonly agreementWaitingForApprovalTimeout: number;
readonly api: RequestorApi;
readonly logger?: Logger;
readonly eventTarget?: EventTarget;
readonly httpAgent: Agent;

constructor(public readonly options?: AgreementOptions) {
const apiKey = options?.yagnaOptions?.apiKey || EnvUtils.getYagnaAppKey();
if (!apiKey) throw new Error("Api key not defined");
const basePath = options?.yagnaOptions?.basePath || EnvUtils.getYagnaApiUrl();
this.httpAgent = new Agent({ keepAlive: true });
const apiConfig = new Configuration({
apiKey,
basePath: `${basePath}/market-api/v1`,
accessToken: apiKey,
baseOptions: { httpAgent: this.httpAgent },
});
this.api = new RequestorApi(apiConfig);
this.agreementRequestTimeout = options?.agreementRequestTimeout || DEFAULTS.agreementRequestTimeout;
this.agreementWaitingForApprovalTimeout =
options?.agreementWaitingForApprovalTimeout || DEFAULTS.agreementWaitingForApprovalTimeout;
Expand Down
Loading
Loading