diff --git a/packages/boot/test/orchestration/contract-upgrade.test.ts b/packages/boot/test/orchestration/contract-upgrade.test.ts index c8dcc03f3c9..7f453ed990d 100644 --- a/packages/boot/test/orchestration/contract-upgrade.test.ts +++ b/packages/boot/test/orchestration/contract-upgrade.test.ts @@ -24,8 +24,8 @@ test.after.always(t => t.context.shutdown?.()); * Because the send-anywhere flow requires a lookup(), it waits forever. This * gives us a point at which we can upgrade the vat with a working agoricNames * and see that the flow continues from that point. (The lookup call is not made - * directly in a flow, but instead from a host API which uses the retriable - * helper. As such it tests both the idempotent retry mechanism of retriable on + * directly in a flow, but instead from a host API which uses the retryable + * helper. As such it tests both the idempotent retry mechanism of retryable on * upgrades, and the ability to resume an async-flow for which a host vow * settles after an upgrade.) */ diff --git a/packages/internal/src/upgrade-api.js b/packages/internal/src/upgrade-api.js index c2cadd7be16..6c35963d780 100644 --- a/packages/internal/src/upgrade-api.js +++ b/packages/internal/src/upgrade-api.js @@ -45,5 +45,29 @@ harden(makeUpgradeDisconnection); * @returns {reason is UpgradeDisconnection} */ export const isUpgradeDisconnection = reason => - isFrozen(reason) && matches(reason, UpgradeDisconnectionShape); + reason != null && // eslint-disable-line eqeqeq + isFrozen(reason) && + matches(reason, UpgradeDisconnectionShape); harden(isUpgradeDisconnection); + +/** + * Returns whether a reason is a 'vat terminated' error generated when an object + * is abandoned by a vat during an upgrade. + * + * Normally we do not want to rely on the `message` of an error object, but this + * is a pragmatic solution to the current state of vat upgrade errors. In the + * future we'd prefer having an error with a cause referencing a disconnection + * object like for promise rejections. See + * https://github.com/Agoric/agoric-sdk/issues/9582 + * + * @param {any} reason + * @returns {reason is Error} + */ +export const isAbandonedError = reason => + reason != null && // eslint-disable-line eqeqeq + isFrozen(reason) && + matches(reason, M.error()) && + // We're not using a constant here since this special value is already + // sprinkled throughout the SDK + reason.message === 'vat terminated'; +harden(isAbandonedError); diff --git a/packages/internal/test/upgrade-api.test.js b/packages/internal/test/upgrade-api.test.js index 5ffc38f9b29..f8eee5d2d92 100644 --- a/packages/internal/test/upgrade-api.test.js +++ b/packages/internal/test/upgrade-api.test.js @@ -1,8 +1,11 @@ // @ts-check import test from 'ava'; +import { makeMarshal } from '@endo/marshal'; + import { makeUpgradeDisconnection, isUpgradeDisconnection, + isAbandonedError, } from '../src/upgrade-api.js'; test('isUpgradeDisconnection must recognize disconnection objects', t => { @@ -18,3 +21,15 @@ test('isUpgradeDisconnection must recognize original-format disconnection object }); t.true(isUpgradeDisconnection(disconnection)); }); + +test('isAbandonedError recognizes marshalled vat terminated errors', t => { + const { fromCapData, toCapData } = makeMarshal(undefined, undefined, { + serializeBodyFormat: 'smallcaps', + errorIdNum: 70_000, + marshalSaveError: () => {}, + }); + const error = harden(Error('vat terminated')); + const remoteError = fromCapData(toCapData(error)); + + t.true(isAbandonedError(remoteError)); +}); diff --git a/packages/orchestration/src/exos/chain-hub.js b/packages/orchestration/src/exos/chain-hub.js index e9f52972e4b..97e25d5d1fc 100644 --- a/packages/orchestration/src/exos/chain-hub.js +++ b/packages/orchestration/src/exos/chain-hub.js @@ -205,7 +205,7 @@ export const makeChainHub = (zone, agoricNames, vowTools) => { valueShape: M.string(), }); - const lookupChainInfo = vowTools.retriable( + const lookupChainInfo = vowTools.retryable( zone, 'lookupChainInfo', /** @param {string} chainName */ @@ -227,7 +227,7 @@ export const makeChainHub = (zone, agoricNames, vowTools) => { }, ); - const lookupConnectionInfo = vowTools.retriable( + const lookupConnectionInfo = vowTools.retryable( zone, 'lookupConnectionInfo', /** @@ -258,7 +258,7 @@ export const makeChainHub = (zone, agoricNames, vowTools) => { ); /* eslint-disable no-use-before-define -- chainHub defined below */ - const lookupChainsAndConnection = vowTools.retriable( + const lookupChainsAndConnection = vowTools.retryable( zone, 'lookupChainsAndConnection', /** diff --git a/packages/orchestration/src/utils/zoe-tools.js b/packages/orchestration/src/utils/zoe-tools.js index e81a1136e3c..a5cbbe389f8 100644 --- a/packages/orchestration/src/utils/zoe-tools.js +++ b/packages/orchestration/src/utils/zoe-tools.js @@ -80,7 +80,7 @@ export const makeZoeTools = (zcf, { when, allVows, allSettled, asVow }) => { // const { zcfSeat: tempSeat, userSeat: userSeatP } = // zcf.makeEmptySeatKit(); // const uSeat = await userSeatP; - // // TODO how do I store in the place for this retriable? + // // TODO how do I store in the place for this retryable? // atomicTransfer(zcf, srcSeat, tempSeat, amounts); // tempSeat.exit(); // return uSeat; diff --git a/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.md b/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.md index 6d091f7bb4f..38ac9729bdd 100644 --- a/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.md +++ b/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.md @@ -31,6 +31,9 @@ Generated by [AVA](https://avajs.dev). chainInfos: {}, connectionInfos: {}, denom: {}, + lookupChainInfo_kindHandle: 'Alleged: kind', + lookupChainsAndConnection_kindHandle: 'Alleged: kind', + lookupConnectionInfo_kindHandle: 'Alleged: kind', }, contract: { 'ChainHub Admin_kindHandle': 'Alleged: kind', @@ -74,8 +77,11 @@ Generated by [AVA](https://avajs.dev). }, }, vows: { + AdminRetryableFlow_kindHandle: 'Alleged: kind', + AdminRetryableFlow_singleton: 'Alleged: AdminRetryableFlow', PromiseWatcher_kindHandle: 'Alleged: kind', VowInternalsKit_kindHandle: 'Alleged: kind', WatchUtils_kindHandle: 'Alleged: kind', + retryableFlowForOutcomeVow: {}, }, } diff --git a/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.snap b/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.snap index 0149838132a..897c3c4e64b 100644 Binary files a/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.snap and b/packages/orchestration/test/examples/snapshots/send-anywhere.test.ts.snap differ diff --git a/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.md b/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.md index afcd6ed2fd1..8d65dc66fcb 100644 --- a/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.md +++ b/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.md @@ -96,6 +96,9 @@ Generated by [AVA](https://avajs.dev). chainName: 'agoric', }, }, + lookupChainInfo_kindHandle: 'Alleged: kind', + lookupChainsAndConnection_kindHandle: 'Alleged: kind', + lookupConnectionInfo_kindHandle: 'Alleged: kind', }, contract: { 'ChainHub Admin_kindHandle': 'Alleged: kind', @@ -208,8 +211,11 @@ Generated by [AVA](https://avajs.dev). }, }, vows: { + AdminRetryableFlow_kindHandle: 'Alleged: kind', + AdminRetryableFlow_singleton: 'Alleged: AdminRetryableFlow', PromiseWatcher_kindHandle: 'Alleged: kind', VowInternalsKit_kindHandle: 'Alleged: kind', WatchUtils_kindHandle: 'Alleged: kind', + retryableFlowForOutcomeVow: {}, }, } diff --git a/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.snap b/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.snap index 406237967cd..45b56186c15 100644 Binary files a/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.snap and b/packages/orchestration/test/examples/snapshots/staking-combinations.test.ts.snap differ diff --git a/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.md b/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.md index 2be729c04fe..59ebbc6aebb 100644 --- a/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.md +++ b/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.md @@ -114,6 +114,9 @@ Generated by [AVA](https://avajs.dev). }, }, denom: {}, + lookupChainInfo_kindHandle: 'Alleged: kind', + lookupChainsAndConnection_kindHandle: 'Alleged: kind', + lookupConnectionInfo_kindHandle: 'Alleged: kind', }, contract: { orchestration: { @@ -156,8 +159,11 @@ Generated by [AVA](https://avajs.dev). }, }, vows: { + AdminRetryableFlow_kindHandle: 'Alleged: kind', + AdminRetryableFlow_singleton: 'Alleged: AdminRetryableFlow', PromiseWatcher_kindHandle: 'Alleged: kind', VowInternalsKit_kindHandle: 'Alleged: kind', WatchUtils_kindHandle: 'Alleged: kind', + retryableFlowForOutcomeVow: {}, }, } diff --git a/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.snap b/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.snap index 4edb9b3096e..c768cf53a18 100644 Binary files a/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.snap and b/packages/orchestration/test/examples/snapshots/unbond.contract.test.ts.snap differ diff --git a/packages/swingset-liveslots/tools/prepare-strict-test-env.js b/packages/swingset-liveslots/tools/prepare-strict-test-env.js index c785f376d34..c93490f5f60 100644 --- a/packages/swingset-liveslots/tools/prepare-strict-test-env.js +++ b/packages/swingset-liveslots/tools/prepare-strict-test-env.js @@ -21,6 +21,8 @@ export { flushIncarnation }; export { eventLoopIteration as nextCrank }; /** + * @import { PromiseKit } from '@endo/promise-kit' + * @import { Baggage } from '@agoric/swingset-liveslots' * @import { ReincarnateOptions } from './setup-vat-data.js' */ @@ -37,6 +39,7 @@ export const annihilate = (options = {}) => { return incarnation; }; +/** @returns {Baggage} */ export const getBaggage = () => { return incarnation.fakeVomKit.cm.provideBaggage(); }; @@ -51,7 +54,7 @@ export const nextLife = (fromIncarnation = incarnation) => { }; /** - * @template {(baggage: import('@agoric/swingset-liveslots').Baggage) => Promise | any} B + * @template {(baggage: Baggage) => Promise | any} B * @param {B} build * @param {(tools: Awaited>) => Promise | void} [run] * @param {object} [options] @@ -72,7 +75,7 @@ export const startLife = async ( oldIncarnationNumber, ); const { fakeVomKit } = nextLife(fromIncarnation); - /** @type {Map>} */ + /** @type {Map>} */ const previouslyWatchedPromises = new Map(); let buildTools; try { diff --git a/packages/vow/README.md b/packages/vow/README.md index 08d32ef3d13..bd8478a8b1b 100644 --- a/packages/vow/README.md +++ b/packages/vow/README.md @@ -150,8 +150,8 @@ Here is an (oversimplified) algorithm that `watch` and `when` use to obtain a final result: ```js -// Directly await the non-retriable original specimen. -// This is non-retriable because we don't know how our caller obtained +// Directly await the non-retryable original specimen. +// This is non-retryable because we don't know how our caller obtained // it in the first place, since it is an application-specific detail // that may not be side-effect free. let result = await specimenP; diff --git a/packages/vow/src/retryable.js b/packages/vow/src/retryable.js new file mode 100644 index 00000000000..98e5416bcb7 --- /dev/null +++ b/packages/vow/src/retryable.js @@ -0,0 +1,224 @@ +import { Fail } from '@endo/errors'; +import { M } from '@endo/patterns'; +import { PromiseWatcherI } from '@agoric/base-zone'; +import { makeAsVow, toPassableCap, VowShape } from './vow-utils.js'; + +/** + * @import {MapStore, WeakMapStore} from '@agoric/store' + * @import {Zone} from '@agoric/base-zone' + * @import {Vow, VowKit, IsRetryableReason, VowTools} from './types.js' + * @import {Passable, PassableCap} from '@endo/pass-style' + */ + +/** + * @typedef {object} PreparationOptions + * @property {() => VowKit} makeVowKit + * @property {IsRetryableReason} isRetryableReason + */ + +/** + * @typedef {(...args: Passable[]) => Promise} RetryableFunc + */ + +const { defineProperties } = Object; + +const RetryableFlowIKit = harden({ + flow: M.interface('Flow', { + restart: M.call().returns(), + getOutcome: M.call().returns(VowShape), + }), + resultWatcher: PromiseWatcherI, +}); + +const AdminRetryableFlowI = M.interface('RetryableFlowAdmin', { + getFlowForOutcomeVow: M.call(VowShape).returns(M.opt(M.remotable('flow'))), +}); + +/** + * @param {Zone} outerZone + * @param {PreparationOptions} outerOptions + */ +export const prepareRetryableTools = (outerZone, outerOptions) => { + const { makeVowKit, isRetryableReason } = outerOptions; + + const asVow = makeAsVow(makeVowKit); + + /** + * So we can give out wrapper functions easily and recover flow objects + * for their activations later. + */ + const flowForOutcomeVowKey = + /** @type {MapStore} */ ( + outerZone.mapStore('retryableFlowForOutcomeVow', { + keyShape: M.remotable('toPassableCap'), + valueShape: M.remotable('flow'), // isDone === false + }) + ); + + /** + * @param {Zone} zone + * @param {string} tag + * @param {RetryableFunc} retryableFunc + */ + const prepareRetryableFlowKit = (zone, tag, retryableFunc) => { + typeof retryableFunc === 'function' || + Fail`retryableFunc must be a callable function ${retryableFunc}`; + + const internalMakeRetryableFlowKit = zone.exoClassKit( + tag, + RetryableFlowIKit, + activationArgs => { + harden(activationArgs); + + return { + activationArgs, // restarting the retryable function uses the original args + outcomeKit: makeVowKit(), // outcome of activation as vow + lastRetryReason: undefined, + runs: 0n, + isDone: false, // persistently done + }; + }, + { + flow: { + /** + * Calls the retryable function, either for the initial run or when + * the result of the previous run fails with a retryable reason. + */ + restart() { + const { state, facets } = this; + const { activationArgs, isDone } = state; + const { flow, resultWatcher } = facets; + + !isDone || + // separate line so I can set a breakpoint + Fail`Cannot restart a done retryable flow ${flow}`; + + const runId = state.runs + 1n; + state.runs = runId; + + let resultP; + try { + resultP = Promise.resolve(retryableFunc(...activationArgs)); + } catch (err) { + resultP = Promise.reject(err); + } + + outerZone.watchPromise(harden(resultP), resultWatcher, runId); + }, + getOutcome() { + const { state } = this; + const { outcomeKit } = state; + return outcomeKit.vow; + }, + }, + resultWatcher: { + onFulfilled(value, runId) { + const { state } = this; + const { runs, outcomeKit } = state; + if (runId !== runs) return; + !state.isDone || + Fail`Cannot resolve a done retryable flow ${this.facets.flow}`; + outcomeKit.resolver.resolve(value); + flowForOutcomeVowKey.delete(toPassableCap(outcomeKit.vow)); + state.isDone = true; + }, + onRejected(reason, runId) { + const { state } = this; + const { runs, outcomeKit } = state; + if (runId !== runs) return; + !state.isDone || + Fail`Cannot reject a done retryable flow ${this.facets.flow}`; + const retryReason = isRetryableReason( + reason, + state.lastRetryReason, + ); + if (retryReason) { + state.lastRetryReason = retryReason; + this.facets.flow.restart(); + } else { + outcomeKit.resolver.reject(reason); + flowForOutcomeVowKey.delete(toPassableCap(outcomeKit.vow)); + state.isDone = true; + } + }, + }, + }, + ); + const makeRetryableFlowKit = activationArgs => { + const retryableKit = internalMakeRetryableFlowKit(activationArgs); + const { flow } = retryableKit; + + const vow = flow.getOutcome(); + flowForOutcomeVowKey.init(toPassableCap(vow), flow); + flow.restart(); + return retryableKit; + }; + return harden(makeRetryableFlowKit); + }; + + /** + * @type {VowTools['retryable']} + */ + const retryable = (zone, tag, retryableFunc) => { + const makeRetryableKit = prepareRetryableFlowKit(zone, tag, retryableFunc); + const wrapperFuncName = `${tag}_retryable`; + + const wrapperFunc = { + /** @param {any[]} args */ + [wrapperFuncName](...args) { + // Make sure any error results in a rejected vow + return asVow(() => { + zone.isStorable(harden(args)) || + Fail`retryable arguments must be storable ${args}`; + const { flow } = makeRetryableKit(args); + return flow.getOutcome(); + }); + }, + }[wrapperFuncName]; + defineProperties(wrapperFunc, { + length: { value: retryableFunc.length }, + }); + // @ts-expect-error inferred generic func + return harden(wrapperFunc); + }; + + const adminRetryableFlow = outerZone.exo( + 'AdminRetryableFlow', + AdminRetryableFlowI, + { + /** + * @param {Vow} outcomeVow + */ + getFlowForOutcomeVow(outcomeVow) { + return flowForOutcomeVowKey.get(toPassableCap(outcomeVow)); + }, + }, + ); + + return harden({ + prepareRetryableFlowKit, + adminRetryableFlow, + retryable, + }); +}; +harden(prepareRetryableTools); + +/** + * @typedef {ReturnType} RetryableTools + */ + +/** + * @typedef {RetryableTools['adminRetryableFlow']} AdminRetryableFlow + */ + +/** + * @typedef {ReturnType} MakeRetryableFlowKit + */ + +/** + * @typedef {ReturnType} RetryableFlowKit + */ + +/** + * @typedef {RetryableFlowKit['flow']} RetryableFlow + */ diff --git a/packages/vow/src/tools.js b/packages/vow/src/tools.js index 1764483e56b..35511de09c9 100644 --- a/packages/vow/src/tools.js +++ b/packages/vow/src/tools.js @@ -3,6 +3,7 @@ import { makeAsVow } from './vow-utils.js'; import { prepareVowKit } from './vow.js'; import { prepareWatchUtils } from './watch-utils.js'; import { prepareWatch } from './watch.js'; +import { prepareRetryableTools } from './retryable.js'; import { makeWhen } from './when.js'; /** @@ -35,25 +36,10 @@ export const prepareBasicVowTools = (zone, powers = {}) => { const watchUtils = makeWatchUtils(); const asVow = makeAsVow(makeVowKit); - // FIXME in https://github.com/Agoric/agoric-sdk/pull/9785 - /** - * @alpha Not yet implemented - * - * Create a function that retries the given function if the underlying - * functions rejects due to upgrade disconnection. - * - * @template {(...args: any[]) => Promise} F - * @param {Zone} fnZone - the zone for the named function - * @param {string} name - * @param {F} fn - * @returns {F extends (...args: infer Args) => Promise ? (...args: Args) => Vow : never} - */ - const retriable = - (fnZone, name, fn) => - // @ts-expect-error cast - (...args) => { - return watch(fn(...args)); - }; + const { retryable } = prepareRetryableTools(zone, { + makeVowKit, + isRetryableReason, + }); /** * Vow-tolerant implementation of Promise.all that takes an iterable of vows @@ -95,7 +81,8 @@ export const prepareBasicVowTools = (zone, powers = {}) => { allSettled, asVow, asPromise, - retriable, + retryable, + retriable: retryable, // For temporary backwards compat with alpha implementation }); }; harden(prepareBasicVowTools); diff --git a/packages/vow/src/types.ts b/packages/vow/src/types.ts index dee2d14531f..7c7e8418b20 100644 --- a/packages/vow/src/types.ts +++ b/packages/vow/src/types.ts @@ -100,6 +100,27 @@ export type AsPromiseFunction< watcherArgs?: C | undefined, ) => Promise; +export interface RetryableTool { + /** + * Create a function that retries the given function if the underlying + * async function rejects due to an upgrade disconnection. The return value + * of the created function is a vow that settles to the final retry result. + * + * The retried function should be idempotent. + * + * @param fnZone the zone for the named function + * @param name base name to use in the zone + * @param fn the retried function + */ + Promise>( + fnZone: Zone, + name: string, + fn: F, + ): F extends (...args: infer Args) => Promise + ? (...args: Args) => Vow + : never; +} + export type VowTools = { /** * Vow-tolerant implementation of Promise.all that takes an iterable of vows @@ -142,13 +163,11 @@ export type VowTools = { fn: (...args: any[]) => Vow> | Awaited | PromiseVow, ) => Vow>; makeVowKit: () => VowKit; - retriable: Promise>( - fnZone: Zone, - name: string, - fn: F, - ) => F extends (...args: infer Args) => Promise - ? (...args: Args) => Vow - : never; + retryable: RetryableTool; + /** + * @deprecated use `retryable` + */ + retriable: RetryableTool; watch: ( specimenP: EVow, watcher?: Watcher | undefined, diff --git a/packages/vow/src/vow-utils.js b/packages/vow/src/vow-utils.js index e18465ce367..d0a40de1694 100644 --- a/packages/vow/src/vow-utils.js +++ b/packages/vow/src/vow-utils.js @@ -29,7 +29,7 @@ harden(isVow); /** * A vow is a passable tagged as 'Vow'. Its payload is a record with * API-versioned remotables. payload.vowV0 is the API for the `watch` and - * `when` operators to use for retriable shortening of the vow chain. + * `when` operators to use for retryable shortening of the vow chain. * * If the specimen is a Vow, return its payload, otherwise undefined. * diff --git a/packages/vow/test/retryable-restart.test.js b/packages/vow/test/retryable-restart.test.js new file mode 100644 index 00000000000..39ca98ac3a6 --- /dev/null +++ b/packages/vow/test/retryable-restart.test.js @@ -0,0 +1,110 @@ +import { + annihilate, + getBaggage, + nextCrank, + startLife, + test, +} from '@agoric/swingset-vat/tools/prepare-strict-test-env-ava.js'; + +import { Fail } from '@endo/errors'; +import { makeDurableZone } from '@agoric/zone/durable.js'; + +import { prepareVowTools } from '../vat.js'; + +test.serial('retries on disconnection', async t => { + annihilate(); + + t.plan(1); + + await startLife( + async baggage => { + const zone = makeDurableZone(baggage, 'durableRoot'); + const { retryable, watch } = prepareVowTools(zone); + const retry = retryable(zone, 'retry', async () => { + // Never resolves, simulates external call + await new Promise(() => {}); + }); + + const watcher = zone.exo('DurableVowTestWatcher', undefined, { + onFulfilled(value) { + t.fail( + `First incarnation watcher onFulfilled triggered with value ${value}`, + ); + }, + onRejected(reason) { + t.fail( + `First incarnation watcher onRejected triggered with reason ${reason}`, + ); + }, + }); + + return { zone, watch, retry, watcher }; + }, + async ({ zone, watch, retry, watcher }) => { + const result = retry(); + zone.makeOnce('result', () => result); + watch(result, watcher); + await nextCrank(); + }, + ); + + await startLife( + baggage => { + const zone = makeDurableZone(baggage, 'durableRoot'); + const { retryable, when } = prepareVowTools(zone); + + // Reconnect retryable definition + retryable(zone, 'retry', async () => { + // Simulate call that settles + await nextCrank(); + return 42; + }); + + zone.exo('DurableVowTestWatcher', undefined, { + onFulfilled(value) { + t.is(value, 42, 'vow resolved with value 42'); + }, + onRejected(reason) { + t.fail( + `Second incarnation watcher onRejected triggered with reason ${reason}`, + ); + }, + }); + + return { zone, when }; + }, + async ({ zone, when }) => { + const result = zone.makeOnce('result', () => Fail`result should exist`); + + await when(result); + }, + ); +}); + +test.serial('errors on non durably storable arguments', async t => { + annihilate(); + + const baggage = getBaggage(); + const zone = makeDurableZone(baggage, 'durableRoot'); + const { retryable, when } = prepareVowTools(zone); + + const passthrough = retryable(zone, 'passthrough', async arg => arg); + + const nonStorableArg = { + promise: new Promise(() => {}), + }; + + t.false(zone.isStorable(nonStorableArg), 'arg is actually non storable'); + + let resultV; + t.notThrows(() => { + resultV = passthrough(nonStorableArg); + }, 'retryable does not synchronously error'); + + const resultP = when(resultV); + await t.throwsAsync( + resultP, + { message: /^retryable arguments must be storable/ }, + 'expected rejection', + ); +}); diff --git a/packages/vow/test/retryable.test.js b/packages/vow/test/retryable.test.js new file mode 100644 index 00000000000..5c9e98089b2 --- /dev/null +++ b/packages/vow/test/retryable.test.js @@ -0,0 +1,196 @@ +// @ts-check +import test from 'ava'; + +import { Fail } from '@endo/errors'; +import { Far } from '@endo/pass-style'; +import { makeHeapZone } from '@agoric/base-zone/heap.js'; +import { eventLoopIteration } from '@agoric/internal/src/testing-utils.js'; + +import { prepareVowKit } from '../src/vow.js'; +import { isVow } from '../src/vow-utils.js'; +import { prepareRetryableTools } from '../src/retryable.js'; +import { makeWhen } from '../src/when.js'; + +/** + * @import {IsRetryableReason} from '../src/types.js' + */ + +/** + * @param {object} [options] + * @param {IsRetryableReason} [options.isRetryableReason] + */ +const makeTestTools = ({ isRetryableReason = () => false } = {}) => { + const zone = makeHeapZone(); + const makeVowKit = prepareVowKit(zone); + const when = makeWhen(isRetryableReason); + + const { retryable, adminRetryableFlow } = prepareRetryableTools(zone, { + makeVowKit, + isRetryableReason, + }); + + return { zone, when, makeVowKit, retryable, adminRetryableFlow }; +}; + +test('successful flow', async t => { + const { zone, when, retryable } = makeTestTools(); + + const succeed = retryable(zone, 'succeed', async () => 42); + + const resultV = succeed(); + const result = await when(resultV); + t.is(result, 42, 'expected result'); +}); + +test('rejected flow', async t => { + const { zone, when, retryable } = makeTestTools(); + + const reject = retryable(zone, 'reject', async () => Fail`some error`); + + const resultV = reject(); + const resultP = when(resultV); + await t.throwsAsync(resultP, { message: 'some error' }, 'expected rejection'); +}); + +test('throwing flow', async t => { + const { zone, when, retryable } = makeTestTools(); + + const error = retryable(zone, 'error', () => Fail`some error`); + + const resultV = error(); + const resultP = when(resultV); + await t.throwsAsync(resultP, { message: 'some error' }, 'expected rejection'); +}); + +test('passable arguments', async t => { + const { zone, when, makeVowKit, retryable } = makeTestTools(); + + const argValue = { + remotable: Far('test'), + promise: Promise.resolve(), + vowKit: makeVowKit(), + }; + + const passthrough = retryable(zone, 'passthrough', async arg => arg); + + const resultV = passthrough(argValue); + const result = await when(resultV); + t.deepEqual(result, argValue, 'expected result'); +}); + +test('non-passable arguments', async t => { + const { zone, when, retryable } = makeTestTools(); + + const passthrough = retryable(zone, 'passthrough', async arg => arg); + + const nonPassableArg = harden({ + foo() { + return 'bar'; + }, + }); + + t.false(zone.isStorable(nonPassableArg), 'arg is actually non passable'); + + let resultV; + t.notThrows(() => { + resultV = passthrough(nonPassableArg); + }, 'retryable does not synchronously error'); + + const resultP = when(resultV); + await t.throwsAsync( + resultP, + { message: /^retryable arguments must be storable/ }, + 'expected rejection', + ); +}); + +test('outcome vow', async t => { + const { zone, when, retryable, adminRetryableFlow } = makeTestTools(); + + const succeed = retryable(zone, 'succeed', async () => 42); + + const resultV = succeed(); + + t.true(isVow(resultV), 'retryable result is vow'); + + const flow = adminRetryableFlow.getFlowForOutcomeVow(resultV); + t.truthy(flow, 'flow from outcome vow'); + + t.is(flow.getOutcome(), resultV, 'outcome vow match'); + + const result = await when(resultV); + t.is(result, 42, 'expected result'); + + t.throws( + () => adminRetryableFlow.getFlowForOutcomeVow(resultV), + undefined, + 'outcome vow not found', + ); +}); + +test('retry', async t => { + const { zone, when, retryable } = makeTestTools({ + isRetryableReason: (reason, priorReason) => + reason !== priorReason && reason.startsWith('retry') && reason, + }); + + const expectedCalls = 3; + + let getResultCalled = 0; + const resultProvider = Far('ResultProvider', { + getResult() { + if (getResultCalled < expectedCalls) { + getResultCalled += 1; + } + // eslint-disable-next-line prefer-promise-reject-errors + return Promise.reject(`retry-${getResultCalled}`); + }, + }); + + const resultFromProvider = retryable( + zone, + 'resultFromProvider', + async provider => provider.getResult(), + ); + + const resultV = resultFromProvider(resultProvider); + + const result = await when(resultV).catch(r => r); + t.is( + result, + `retry-${expectedCalls}`, + 'expected getResult called multiple times', + ); +}); + +test('restart', async t => { + const { zone, when, retryable, adminRetryableFlow } = makeTestTools(); + + let runNum = 0; + const restarted = retryable(zone, 'testRestartedRetryable', async () => { + // Non idempotent function to simplify the test + runNum += 1; + const currentRun = runNum; + await eventLoopIteration(); + if (currentRun < 3) { + // Trigger our own invocation restart + // eslint-disable-next-line no-use-before-define + flow.restart(); + } + if (currentRun === 2) { + throw Error('reject'); + } + return currentRun; + }); + + const resultV = restarted(); + const flow = adminRetryableFlow.getFlowForOutcomeVow(resultV); + t.truthy(flow, 'flow from outcome vow'); + + const result = await when(resultV); + t.is(result, 3, 'flow result from restart'); + + t.throws(() => flow.restart(), { + message: /^Cannot restart a done retryable flow/, + }); +}); diff --git a/packages/vow/test/types.test-d.ts b/packages/vow/test/types.test-d.ts index 0993fe14e0b..d609bdac4b9 100644 --- a/packages/vow/test/types.test-d.ts +++ b/packages/vow/test/types.test-d.ts @@ -8,11 +8,11 @@ const vt: VowTools = null as any; const zone: Zone = null as any; // @ts-expect-error function param must return promise -vt.retriable(zone, 'foo', () => null); -vt.retriable(zone, 'foo', () => Promise.resolve(null)); +vt.retryable(zone, 'foo', () => null); +vt.retryable(zone, 'foo', () => Promise.resolve(null)); expectType<(p1: number, p2: string) => Vow<{ someValue: 'bar' }>>( - vt.retriable(zone, 'foo', (p1: number, p2: string) => + vt.retryable(zone, 'foo', (p1: number, p2: string) => Promise.resolve({ someValue: 'bar' } as const), ), ); diff --git a/packages/vow/vat.js b/packages/vow/vat.js index a486730b9c4..ad7d177d409 100644 --- a/packages/vow/vat.js +++ b/packages/vow/vat.js @@ -5,7 +5,10 @@ /* global globalThis */ // @ts-check -import { isUpgradeDisconnection } from '@agoric/internal/src/upgrade-api.js'; +import { + isUpgradeDisconnection, + isAbandonedError, +} from '@agoric/internal/src/upgrade-api.js'; import { makeHeapZone } from '@agoric/base-zone/heap.js'; import { prepareBasicVowTools } from './src/tools.js'; @@ -15,11 +18,16 @@ import makeE from './src/E.js'; const isRetryableReason = (reason, priorRetryValue) => { if ( isUpgradeDisconnection(reason) && - (!priorRetryValue || + (!isUpgradeDisconnection(priorRetryValue) || reason.incarnationNumber > priorRetryValue.incarnationNumber) ) { return reason; } + // For abandoned errors there is no way to differentiate errors from + // consecutive upgrades + if (isAbandonedError(reason) && !isAbandonedError(priorRetryValue)) { + return reason; + } return undefined; }; diff --git a/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.md b/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.md index 0d19048e384..feb090d21fe 100644 --- a/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.md +++ b/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.md @@ -9,11 +9,14 @@ Generated by [AVA](https://avajs.dev). > contract baggage after start { + AdminRetryableFlow_kindHandle: 'Alleged: kind', + AdminRetryableFlow_singleton: 'Alleged: AdminRetryableFlow', PromiseWatcher_kindHandle: 'Alleged: kind', VowInternalsKit_kindHandle: 'Alleged: kind', WatchUtils_kindHandle: 'Alleged: kind', publicFacet_kindHandle: 'Alleged: kind', publicFacet_singleton: 'Alleged: publicFacet', + retryableFlowForOutcomeVow: {}, vowResolver: { resolver: Object @Alleged: VowInternalsKit resolver {}, vow: Object @Vow { diff --git a/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.snap b/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.snap index 756b545154f..2631812d4c3 100644 Binary files a/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.snap and b/packages/zoe/test/unitTests/contracts/snapshots/valueVow.test.js.snap differ