From 6ee1df8739fb44727362e2c86be84a1c97f4fc6c Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 27 Mar 2024 10:32:31 -0700 Subject: [PATCH 1/5] refactor(swingset): create vat state non-lazily Previously, vat state initialization (e.g. setting counters to zero) happened lazily, the first time that `provideVatKeeper()` was called. When creating a new vat, the pattern was: vk = kernelKeeper.provideVatKeeper(); vk.setSourceAndOptions(source, options); Now, we initialize both counters and source/options explicitly, in `recordVatOptions`, when the static/dynamic vat is first defined: kernelKeeper.createVatState(vatID, source, options); In the future, this will make it easier for `provideVatKeeper` to rely upon the presence of all vat state keys, especially `options`. Previously, vatKeeper.getOptions() would tolerate a missing key by returning empty options. The one place where this was needed (terminating a barely-created vat because startVat() failed, using getOptions().critical) was changed, so this tolerance is no longer needed, and was removed. The tolerance caused bug #9157 (kernel doesn't panic when it should), which continues to be present, but at least won't cause a crash. refs #8980 --- packages/SwingSet/src/kernel/kernel.js | 11 +++++-- .../SwingSet/src/kernel/state/kernelKeeper.js | 9 ++++-- .../SwingSet/src/kernel/state/vatKeeper.js | 30 +++++++++++++------ packages/SwingSet/src/lib/recordVatOptions.js | 4 +-- packages/SwingSet/test/clist.test.js | 8 +++++ packages/SwingSet/test/state.test.js | 20 ++++++------- 6 files changed, 55 insertions(+), 27 deletions(-) diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js index 200a69a434b..9aa0e8ed8e4 100644 --- a/packages/SwingSet/src/kernel/kernel.js +++ b/packages/SwingSet/src/kernel/kernel.js @@ -252,8 +252,7 @@ export default function buildKernel( */ async function terminateVat(vatID, shouldReject, info) { console.log(`kernel terminating vat ${vatID} (failure=${shouldReject})`); - const vatKeeper = kernelKeeper.provideVatKeeper(vatID); - const critical = vatKeeper.getOptions().critical; + let critical = false; insistCapData(info); // ISSUE: terminate stuff in its own crank like creation? // TODO: if a static vat terminates, panic the kernel? @@ -264,6 +263,14 @@ export default function buildKernel( // check will report 'false'. That's fine, there's no state to // clean up. if (kernelKeeper.vatIsAlive(vatID)) { + // If there was no vat state, we can't make a vatKeeper to ask for + // options. For now, pretend the vat was non-critical. This will fail + // to panic the kernel for startVat failures in critical vats + // (#9157). The fix will add .critical to CrankResults, populated by a + // getOptions query in deliveryCrankResults() or copied from + // dynamicOptions in processCreateVat. + critical = kernelKeeper.provideVatKeeper(vatID).getOptions().critical; + // Reject all promises decided by the vat, making sure to capture the list // of kpids before that data is deleted. const deadPromises = [...kernelKeeper.enumeratePromisesByDecider(vatID)]; diff --git a/packages/SwingSet/src/kernel/state/kernelKeeper.js b/packages/SwingSet/src/kernel/state/kernelKeeper.js index 192be013c41..2853333de8f 100644 --- a/packages/SwingSet/src/kernel/state/kernelKeeper.js +++ b/packages/SwingSet/src/kernel/state/kernelKeeper.js @@ -1288,15 +1288,17 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { maybeFreeKrefs.clear(); } + function createVatState(vatID, source, options) { + initializeVatState(kvStore, transcriptStore, vatID, source, options); + } + function provideVatKeeper(vatID) { insistVatID(vatID); const found = ephemeral.vatKeepers.get(vatID); if (found !== undefined) { return found; } - if (!kvStore.has(`${vatID}.o.nextID`)) { - initializeVatState(kvStore, transcriptStore, vatID); - } + assert(kvStore.has(`${vatID}.o.nextID`), `${vatID} was not initialized`); const vk = makeVatKeeper( kvStore, transcriptStore, @@ -1610,6 +1612,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { getVatIDForName, allocateVatIDForNameIfNeeded, allocateUnusedVatID, + createVatState, provideVatKeeper, vatIsAlive, evictVatKeeper, diff --git a/packages/SwingSet/src/kernel/state/vatKeeper.js b/packages/SwingSet/src/kernel/state/vatKeeper.js index 2d6312c7c4c..481cc06c01c 100644 --- a/packages/SwingSet/src/kernel/state/vatKeeper.js +++ b/packages/SwingSet/src/kernel/state/vatKeeper.js @@ -39,11 +39,30 @@ const FIRST_DEVICE_ID = 70n; * @param {*} transcriptStore Accompanying transcript store * @param {string} vatID The vat ID string of the vat in question * TODO: consider making this part of makeVatKeeper + * @param {SourceOfBundle} source + * @param {RecordedVatOptions} options */ -export function initializeVatState(kvStore, transcriptStore, vatID) { +export function initializeVatState( + kvStore, + transcriptStore, + vatID, + source, + options, +) { + assert(options.workerOptions, `vat ${vatID} options missing workerOptions`); + assert(source); + assert('bundle' in source || 'bundleName' in source || 'bundleID' in source); + assert.typeof(options, 'object'); + const count = options.reapInterval; + assert(count === 'never' || isNat(count), `bad reapCountdown ${count}`); + kvStore.set(`${vatID}.o.nextID`, `${FIRST_OBJECT_ID}`); kvStore.set(`${vatID}.p.nextID`, `${FIRST_PROMISE_ID}`); kvStore.set(`${vatID}.d.nextID`, `${FIRST_DEVICE_ID}`); + kvStore.set(`${vatID}.source`, JSON.stringify(source)); + kvStore.set(`${vatID}.options`, JSON.stringify(options)); + kvStore.set(`${vatID}.reapInterval`, `${count}`); + kvStore.set(`${vatID}.reapCountdown`, `${count}`); transcriptStore.initTranscript(vatID); } @@ -125,16 +144,10 @@ export function makeVatKeeper( function getOptions() { /** @type { RecordedVatOptions } */ - const options = JSON.parse(kvStore.get(`${vatID}.options`) || '{}'); + const options = JSON.parse(getRequired(`${vatID}.options`)); return harden(options); } - function initializeReapCountdown(count) { - count === 'never' || isNat(count) || Fail`bad reapCountdown ${count}`; - kvStore.set(`${vatID}.reapInterval`, `${count}`); - kvStore.set(`${vatID}.reapCountdown`, `${count}`); - } - function updateReapInterval(reapInterval) { reapInterval === 'never' || isNat(reapInterval) || @@ -656,7 +669,6 @@ export function makeVatKeeper( setSourceAndOptions, getSourceAndOptions, getOptions, - initializeReapCountdown, countdownToReap, updateReapInterval, nextDeliveryNum, diff --git a/packages/SwingSet/src/lib/recordVatOptions.js b/packages/SwingSet/src/lib/recordVatOptions.js index 09b8e9c759a..658b5f9f713 100644 --- a/packages/SwingSet/src/lib/recordVatOptions.js +++ b/packages/SwingSet/src/lib/recordVatOptions.js @@ -39,9 +39,7 @@ export const makeVatOptionRecorder = (kernelKeeper, bundleHandler) => { critical, meterID, }); - const vatKeeper = kernelKeeper.provideVatKeeper(vatID); - vatKeeper.setSourceAndOptions(source, vatOptions); - vatKeeper.initializeReapCountdown(vatOptions.reapInterval); + kernelKeeper.createVatState(vatID, source, vatOptions); }; /** diff --git a/packages/SwingSet/test/clist.test.js b/packages/SwingSet/test/clist.test.js index 2b3831da74c..10237c76322 100644 --- a/packages/SwingSet/test/clist.test.js +++ b/packages/SwingSet/test/clist.test.js @@ -13,6 +13,9 @@ test(`clist reachability`, async t => { const s = kk.kvStore; kk.createStartingKernelState({ defaultManagerType: 'local' }); const vatID = kk.allocateUnusedVatID(); + const source = { bundleID: 'foo' }; + const options = { workerOptions: 'foo', reapInterval: 1 }; + kk.createVatState(vatID, source, options); const vk = kk.provideVatKeeper(vatID); const ko1 = kk.addKernelObject('v1', 1); @@ -98,12 +101,17 @@ test('getImporters', async t => { kk.createStartingKernelState({ defaultManagerType: 'local' }); const vatID1 = kk.allocateUnusedVatID(); + const source = { bundleID: 'foo' }; + const options = { workerOptions: 'foo', reapInterval: 1 }; + kk.createVatState(vatID1, source, options); kk.addDynamicVatID(vatID1); const vk1 = kk.provideVatKeeper(vatID1); const vatID2 = kk.allocateUnusedVatID(); + kk.createVatState(vatID2, source, options); kk.addDynamicVatID(vatID2); const vk2 = kk.provideVatKeeper(vatID2); const vatID3 = kk.allocateUnusedVatID(); + kk.createVatState(vatID3, source, options); kk.addDynamicVatID(vatID3); const vk3 = kk.provideVatKeeper(vatID3); diff --git a/packages/SwingSet/test/state.test.js b/packages/SwingSet/test/state.test.js index bdabd07f02a..d1ab7b83d50 100644 --- a/packages/SwingSet/test/state.test.js +++ b/packages/SwingSet/test/state.test.js @@ -511,8 +511,11 @@ test('vatKeeper', async t => { const store = buildKeeperStorageInMemory(); const k = makeKernelKeeper(store, null); k.createStartingKernelState({ defaultManagerType: 'local' }); - const v1 = k.allocateVatIDForNameIfNeeded('name1'); + const source = { bundleID: 'foo' }; + const options = { workerOptions: 'foo', reapInterval: 1 }; + k.createVatState(v1, source, options); + const vk = k.provideVatKeeper(v1); // TODO: confirm that this level of caching is part of the API t.is(vk, k.provideVatKeeper(v1)); @@ -547,18 +550,15 @@ test('vatKeeper.getOptions', async t => { const store = buildKeeperStorageInMemory(); const k = makeKernelKeeper(store, null); k.createStartingKernelState({ defaultManagerType: 'local' }); - const v1 = k.allocateVatIDForNameIfNeeded('name1'); - const vk = k.provideVatKeeper(v1); const bundleID = 'b1-00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'; - vk.setSourceAndOptions( - { bundleID }, - { - workerOptions: { type: 'local' }, - name: 'fred', - }, - ); + const source = { bundleID }; + const workerOptions = { type: 'local' }; + const options = { workerOptions, name: 'fred', reapInterval: 1 }; + k.createVatState(v1, source, options); + + const vk = k.provideVatKeeper(v1); const { name } = vk.getOptions(); t.is(name, 'fred'); }); From ee58485679f90c572499c8033803883a07824ec2 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 27 Mar 2024 10:43:15 -0700 Subject: [PATCH 2/5] fix(swingset): use "dirt" to schedule vat reap/bringOutYourDead NOTE: deployed kernels require a new `upgradeSwingset()` call upon (at least) first boot after upgrading to this version of the kernel code. See below for details. `dispatch.bringOutYourDead()`, aka "reap", triggers garbage collection inside a vat, and gives it a chance to drop imported c-list vrefs that are no longer referenced by anything inside the vat. Previously, each vat has a configurable parameter named `reapInterval`, which defaults to a kernel-wide `defaultReapInterval` (but can be set separately for each vat). This defaults to 1, mainly for unit testing, but real applications set it to something like 1000. This caused BOYD to happen once every 1000 deliveries, plus an extra BOYD just before we save an XS heap-state snapshot. This commit switches to a "dirt"-based BOYD scheduler, wherein we consider the vat to get more and more dirty as it does work, and eventually it reaches a `reapDirtThreshold` that triggers the BOYD (which resets the dirt counter). We continue to track `dirt.deliveries` as before, with the same defaults. But we add a new `dirt.gcKrefs` counter, which is incremented by the krefs we submit to the vat in GC deliveries. For example, calling `dispatch.dropImports([kref1, kref2])` would increase `dirt.gcKrefs` by two. The `reapDirtThreshold.gcKrefs` limit defaults to 20. For normal use patterns, this will trigger a BOYD after ten krefs have been dropped and retired. We choose this value to allow the #8928 slow vat termination process to trigger BOYD frequently enough to keep the BOYD cranks small: since these will be happening constantly (in the "background"), we don't want them to take more than 500ms or so. Given the current size of the large vats that #8928 seeks to terminate, 10 krefs seems like a reasonable limit. And of course we don't want to perform too many BOYDs, so `gcKrefs: 20` is about the smallest threshold we'd want to use. External APIs continue to accept `reapInterval`, and now also accept `reapGCKrefs`, and `neverReap` (a boolean which inhibits all BOYD, even new forms of dirt added in the future). * kernel config record * takes `config.defaultReapInterval` and `defaultReapGCKrefs` * takes `vat.NAME.creationOptions.reapInterval` and `.reapGCKrefs` and `.neverReap` * `controller.changeKernelOptions()` still takes `defaultReapInterval` but now also accepts `defaultReapGCKrefs` The APIs available to userspace code (through `vatAdminSvc`) are unchanged (partially due to upgrade/backwards-compatibility limitations), and continue to only support setting `reapInterval`. Internally, this just modifies `reapDirtThreshold.deliveries`. * `E(vatAdminSvc).createVat(bcap, { reapInterval })` * `E(adminNode).upgrade(bcap, { reapInterval })` * `E(adminNode).changeOptions({ reapInterval })` Internally, the kernel-wide state records `defaultReapDirtThreshold` instead of `defaultReapInterval`, and each vat records `.reapDirtThreshold` in their `vNN.options` key instead of `vNN.reapInterval`. The vat-level records override the kernel-wide values. The current dirt level is recorded in `vNN.reapDirt`. NOTE: deployed kernels require explicit state upgrade, with: ```js import { upgradeSwingset } from '@agoric/swingset-vat'; .. upgradeSwingset(kernelStorage); ``` This must be called after upgrading to the new kernel code/release, and before calling `buildVatController()`. It is safe to call on every reboot (it will only modify the swingstore when the kernel version has changed). If changes are made, the host application is responsible for commiting them, as well as recording any export-data updates (if the host configured the swingstore with an export-data callback). During this upgrade, the old `reapCountdown` value is used to initialize the vat's `reapDirt.deliveries` counter, so the upgrade shouldn't disrupt the existing schedule. Vats which used `reapInterval = 'never'` (eg comms) will get a `reapDirtThreshold.never = true`, so they continue to inhibit BOYD. Any per-vat settings that match the kernel-wide settings are removed, allowing the kernel values to take precedence (as well as changes to the kernel-wide values; i.e. the per-vat settings are not sticky). We do not track dirt when the corresponding threshold is 'never', or if `neverReap` is true, to avoid incrementing the comms dirt counters forever. This design leaves room for adding `.computrons` to the dirt record, as well as tracking a separate `snapshotDirt` counter (to trigger XS heap snapshots, ala #6786). We add `reapDirtThreshold.computrons`, but do not yet expose an API to set it. Future work includes: * upgrade vat-vat-admin to let userspace set `reapDirtThreshold` New tests were added to exercise the upgrade process, and other tests were updated to match the new internal initialization pattern. We now reset the dirt counter upon any BOYD, so this also happens to help with #8665 (doing a `reapAllVats()` resets the delivery counters, so future BOYDs will be delayed, which is what we want). But we should still change `controller.reapAllVats()` to avoid BOYDs on vats which haven't received any deliveries. closes #8980 --- .../src/controller/initializeKernel.js | 39 ++- .../src/controller/initializeSwingset.js | 2 +- .../src/controller/upgradeSwingset.js | 193 +++++++++++++++ packages/SwingSet/src/index.js | 2 +- packages/SwingSet/src/kernel/kernel.js | 100 ++++++-- .../SwingSet/src/kernel/state/kernelKeeper.js | 149 +++++++++--- .../SwingSet/src/kernel/state/vatKeeper.js | 141 ++++++++--- .../src/kernel/vat-loader/manager-factory.js | 1 - .../src/kernel/vat-loader/vat-loader.js | 2 +- packages/SwingSet/src/lib/recordVatOptions.js | 18 +- packages/SwingSet/src/types-external.js | 33 +-- packages/SwingSet/src/types-internal.js | 54 +++- .../test/bundling/bundles-kernel.test.js | 3 +- .../change-parameters.test.js | 31 ++- packages/SwingSet/test/clist.test.js | 8 +- packages/SwingSet/test/controller.test.js | 14 ++ packages/SwingSet/test/kernel.test.js | 194 ++++++++++++++- .../SwingSet/test/snapshots/state.test.js.md | 4 +- .../test/snapshots/state.test.js.snap | Bin 276 -> 276 bytes packages/SwingSet/test/state.test.js | 230 +++++++++++++++++- .../SwingSet/test/upgrade-swingset.test.js | 202 +++++++++++++++ packages/SwingSet/test/vat-admin/bootstrap.js | 8 + .../test/vat-admin/create-vat.test.js | 37 +++ 23 files changed, 1321 insertions(+), 144 deletions(-) create mode 100644 packages/SwingSet/src/controller/upgradeSwingset.js create mode 100644 packages/SwingSet/test/upgrade-swingset.test.js diff --git a/packages/SwingSet/src/controller/initializeKernel.js b/packages/SwingSet/src/controller/initializeKernel.js index e9e26f93411..e0f70a44690 100644 --- a/packages/SwingSet/src/controller/initializeKernel.js +++ b/packages/SwingSet/src/controller/initializeKernel.js @@ -9,14 +9,32 @@ import { insistVatID } from '../lib/id.js'; import { makeVatSlot } from '../lib/parseVatSlots.js'; import { insistStorageAPI } from '../lib/storageAPI.js'; import { makeVatOptionRecorder } from '../lib/recordVatOptions.js'; -import makeKernelKeeper from '../kernel/state/kernelKeeper.js'; +import makeKernelKeeper, { + DEFAULT_DELIVERIES_PER_BOYD, + DEFAULT_GC_KREFS_PER_BOYD, +} from '../kernel/state/kernelKeeper.js'; import { exportRootObject } from '../kernel/kernel.js'; import { makeKernelQueueHandler } from '../kernel/kernelQueue.js'; +/** + * @typedef { import('../types-external.js').SwingSetKernelConfig } SwingSetKernelConfig + * @typedef { import('../types-external.js').SwingStoreKernelStorage } SwingStoreKernelStorage + * @typedef { import('../types-internal.js').InternalKernelOptions } InternalKernelOptions + * @typedef { import('../types-internal.js').ReapDirtThreshold } ReapDirtThreshold + */ + function makeVatRootObjectSlot() { return makeVatSlot('object', true, 0); } +/** + * @param {SwingSetKernelConfig} config + * @param {SwingStoreKernelStorage} kernelStorage + * @param {*} [options] + * @returns {Promise} KPID of the bootstrap message + * result promise + */ + export async function initializeKernel(config, kernelStorage, options = {}) { const { verbose = false, @@ -25,6 +43,9 @@ export async function initializeKernel(config, kernelStorage, options = {}) { const logStartup = verbose ? console.debug : () => 0; insistStorageAPI(kernelStorage.kvStore); + const CURRENT_VERSION = 1; + kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); + const kernelSlog = null; const kernelKeeper = makeKernelKeeper(kernelStorage, kernelSlog); const optionRecorder = makeVatOptionRecorder(kernelKeeper, bundleHandler); @@ -33,14 +54,22 @@ export async function initializeKernel(config, kernelStorage, options = {}) { assert(!wasInitialized); const { defaultManagerType, - defaultReapInterval, + defaultReapInterval = DEFAULT_DELIVERIES_PER_BOYD, + defaultReapGCKrefs = DEFAULT_GC_KREFS_PER_BOYD, relaxDurabilityRules, snapshotInitial, snapshotInterval, } = config; + /** @type { ReapDirtThreshold } */ + const defaultReapDirtThreshold = { + deliveries: defaultReapInterval, + gcKrefs: defaultReapGCKrefs, + computrons: 'never', // TODO no knob? + }; + /** @type { InternalKernelOptions } */ const kernelOptions = { defaultManagerType, - defaultReapInterval, + defaultReapDirtThreshold, relaxDurabilityRules, snapshotInitial, snapshotInterval, @@ -49,7 +78,7 @@ export async function initializeKernel(config, kernelStorage, options = {}) { for (const id of Object.keys(config.idToBundle || {})) { const bundle = config.idToBundle[id]; - assert.equal(bundle.moduleFormat, 'endoZipBase64'); + assert(bundle.moduleFormat === 'endoZipBase64'); if (!kernelKeeper.hasBundle(id)) { kernelKeeper.addBundle(id, bundle); } @@ -86,6 +115,8 @@ export async function initializeKernel(config, kernelStorage, options = {}) { 'useTranscript', 'critical', 'reapInterval', + 'reapGCKrefs', + 'neverReap', 'nodeOptions', ]); const vatID = kernelKeeper.allocateVatIDForNameIfNeeded(name); diff --git a/packages/SwingSet/src/controller/initializeSwingset.js b/packages/SwingSet/src/controller/initializeSwingset.js index 649c68bcd6f..2ea889af3e4 100644 --- a/packages/SwingSet/src/controller/initializeSwingset.js +++ b/packages/SwingSet/src/controller/initializeSwingset.js @@ -397,7 +397,7 @@ export async function initializeSwingset( enableSetup: true, managerType: 'local', useTranscript: false, - reapInterval: 'never', + neverReap: true, }, }; } diff --git a/packages/SwingSet/src/controller/upgradeSwingset.js b/packages/SwingSet/src/controller/upgradeSwingset.js new file mode 100644 index 00000000000..3c194b5ffa5 --- /dev/null +++ b/packages/SwingSet/src/controller/upgradeSwingset.js @@ -0,0 +1,193 @@ +import { + DEFAULT_REAP_DIRT_THRESHOLD_KEY, + DEFAULT_GC_KREFS_PER_BOYD, + getAllDynamicVats, + getAllStaticVats, +} from '../kernel/state/kernelKeeper.js'; + +const upgradeVatV0toV1 = (kvStore, defaultReapDirtThreshold, vatID) => { + // This is called, once per vat, when upgradeSwingset migrates from + // v0 to v1 + + // schema v0: + // Each vat has a `vNN.reapInterval` and `vNN.reapCountdown`. + // vNN.options has a `.reapInterval` property (however it was not + // updated by processChangeVatOptions). Either all are numbers, or + // all are 'never'. + + const oldReapIntervalKey = `${vatID}.reapInterval`; + const oldReapCountdownKey = `${vatID}.reapCountdown`; + const vatOptionsKey = `${vatID}.options`; + + // schema v1: + // Each vat has a `vNN.reapDirt`, and vNN.options has a + // `.reapDirtThreshold` property (which overrides kernel-wide + // `defaultReapDirtThreshold`) + + const reapDirtKey = `${vatID}.reapDirt`; + + assert(kvStore.has(oldReapIntervalKey), oldReapIntervalKey); + assert(kvStore.has(oldReapCountdownKey), oldReapCountdownKey); + assert(!kvStore.has(reapDirtKey), reapDirtKey); + + // initialize or upgrade state + const reapDirt = {}; // all missing keys are treated as zero + const threshold = {}; + + const reapIntervalString = kvStore.get(oldReapIntervalKey); + assert(reapIntervalString !== undefined); + const reapCountdownString = kvStore.get(oldReapCountdownKey); + assert(reapCountdownString !== undefined); + const intervalIsNever = reapIntervalString === 'never'; + const countdownIsNever = reapCountdownString === 'never'; + assert( + (intervalIsNever && countdownIsNever) || + (!intervalIsNever && !countdownIsNever), + `reapInterval=${reapIntervalString}, reapCountdown=${reapCountdownString}`, + ); + + if (!intervalIsNever && !countdownIsNever) { + // deduce delivery count from old countdown values + const reapInterval = Number.parseInt(reapIntervalString, 10); + const reapCountdown = Number.parseInt(reapCountdownString, 10); + const deliveries = reapInterval - reapCountdown; + reapDirt.deliveries = Math.max(deliveries, 0); // just in case + if (reapInterval !== defaultReapDirtThreshold.deliveries) { + threshold.deliveries = reapInterval; + } + } + + // old vats that were never reaped (eg comms) used + // reapInterval='never', so respect that and set the other + // threshold values to never as well + if (intervalIsNever) { + threshold.never = true; + } + kvStore.delete(oldReapIntervalKey); + kvStore.delete(oldReapCountdownKey); + kvStore.set(reapDirtKey, JSON.stringify(reapDirt)); + + // remove .reapInterval from options, replace with .reapDirtThreshold + const options = JSON.parse(kvStore.get(vatOptionsKey)); + delete options.reapInterval; + options.reapDirtThreshold = threshold; + kvStore.set(vatOptionsKey, JSON.stringify(options)); +}; + +/** + * (maybe) upgrade the kernel state to the current schema + * + * This function is responsible for bringing the kernel's portion of + * swing-store (kernelStorage) up to the current version. The host app + * must call this each time it launches with a new version of + * swingset, before using makeSwingsetController() to build the + * kernel/controller (which will throw an error if handed an old + * database). It is ok to call it only on those reboots, but it is + * also safe to call on every reboot, because upgradeSwingset() is a + * no-op if the DB is already up-to-date. + * + * If an upgrade is needed, this function will modify the DB state, so + * the host app must be prepared for export-data callbacks being + * called during the upgrade, and it is responsible for doing a + * `hostStorage.commit()` afterwards. + * + * @param { SwingStoreKernelStorage } kernelStorage + * @returns { boolean } true if any changes were made + */ +export const upgradeSwingset = kernelStorage => { + const { kvStore } = kernelStorage; + let modified = false; + let vstring = kvStore.get('version'); + if (vstring === undefined) { + vstring = '0'; + } + let version = Number(vstring); + + /** + * @param {string} key + * @returns {string} + */ + function getRequired(key) { + if (!kvStore.has(key)) { + throw Error(`storage lacks required key ${key}`); + } + // @ts-expect-error already checked .has() + return kvStore.get(key); + } + + // kernelKeeper.js has a large comment that defines our current + // kvStore schema, with a section describing the deltas. The upgrade + // steps applied here must match. + + // schema v0: + // The kernel overall has `kernel.defaultReapInterval`. + // Each vat has a `vNN.reapInterval` and `vNN.reapCountdown`. + // vNN.options has a `.reapInterval` property (however it was not + // updated by processChangeVatOptions, so do not rely upon its + // value). Either all are numbers, or all are 'never'. + + if (version < 1) { + // schema v1: + // The kernel overall has `kernel.defaultReapDirtThreshold`. + // Each vat has a `vNN.reapDirt`, and vNN.options has a + // `.reapDirtThreshold` property + + // So: + // * replace `kernel.defaultReapInterval` with + // `kernel.defaultReapDirtThreshold` + // * replace vat's `vNN.reapInterval`/`vNN.reapCountdown` with + // `vNN.reapDirt` and a `vNN.reapDirtThreshold` in `vNN.options` + // * then do per-vat upgrades with upgradeVatV0toV1 + + // upgrade from old kernel.defaultReapInterval + + const oldDefaultReapIntervalKey = 'kernel.defaultReapInterval'; + assert(kvStore.has(oldDefaultReapIntervalKey)); + assert(!kvStore.has(DEFAULT_REAP_DIRT_THRESHOLD_KEY)); + + /** + * @typedef { import('../types-internal.js').ReapDirtThreshold } ReapDirtThreshold + */ + + /** @type ReapDirtThreshold */ + const threshold = { + deliveries: 'never', + gcKrefs: 'never', + computrons: 'never', + }; + + const oldValue = getRequired(oldDefaultReapIntervalKey); + if (oldValue !== 'never') { + const value = Number.parseInt(oldValue, 10); + assert.typeof(value, 'number'); + threshold.deliveries = value; + // if BOYD wasn't turned off entirely (eg + // defaultReapInterval='never', which only happens in unit + // tests), then pretend we wanted a gcKrefs= threshold all + // along, so all vats get a retroactive gcKrefs threshold, which + // we need for the upcoming slow-vat-deletion to not trigger + // gigantic BOYD and break the kernel + threshold.gcKrefs = DEFAULT_GC_KREFS_PER_BOYD; + } + harden(threshold); + kvStore.set(DEFAULT_REAP_DIRT_THRESHOLD_KEY, JSON.stringify(threshold)); + kvStore.delete(oldDefaultReapIntervalKey); + + // now upgrade all vats + for (const [_name, vatID] of getAllStaticVats(kvStore)) { + upgradeVatV0toV1(kvStore, threshold, vatID); + } + for (const vatID of getAllDynamicVats(getRequired)) { + upgradeVatV0toV1(kvStore, threshold, vatID); + } + + modified = true; + version = 1; + } + + if (modified) { + kvStore.set('version', `${version}`); + } + return modified; +}; +harden(upgradeSwingset); diff --git a/packages/SwingSet/src/index.js b/packages/SwingSet/src/index.js index 3ba4767ef54..af694b5a215 100644 --- a/packages/SwingSet/src/index.js +++ b/packages/SwingSet/src/index.js @@ -9,7 +9,7 @@ export { loadBasedir, loadSwingsetConfigFile, } from './controller/initializeSwingset.js'; - +export { upgradeSwingset } from './controller/upgradeSwingset.js'; export { buildMailboxStateMap, buildMailbox, diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js index 9aa0e8ed8e4..036f968a7bb 100644 --- a/packages/SwingSet/src/kernel/kernel.js +++ b/packages/SwingSet/src/kernel/kernel.js @@ -364,6 +364,7 @@ export default function buildKernel( * * @typedef { import('@agoric/swingset-liveslots').MeterConsumption } MeterConsumption * @typedef { import('../types-internal.js').MeterID } MeterID + * @typedef { import('../types-internal.js').Dirt } Dirt * * Any delivery crank (send, notify, start-vat.. anything which is allowed * to make vat delivery) emits one of these status events if a delivery @@ -382,7 +383,7 @@ export default function buildKernel( * didDelivery?: VatID, // we made a delivery to a vat, for run policy and save-snapshot * computrons?: BigInt, // computron count for run policy * meterID?: string, // deduct those computrons from a meter - * decrementReapCount?: { vatID: VatID }, // the reap counter should decrement + * measureDirt?: { vatID: VatID, dirt: Dirt }, // dirt counters should increment * terminate?: { vatID: VatID, reject: boolean, info: SwingSetCapData }, // terminate vat, notify vat-admin * vatAdminMethargs?: RawMethargs, // methargs to notify vat-admin about create/upgrade results * } } CrankResults @@ -449,16 +450,17 @@ export default function buildKernel( * event handler. * * Two flags influence this: - * `decrementReapCount` is used for deliveries that run userspace code + * `measureDirt` is used for non-BOYD deliveries * `meterID` means we should check a meter * * @param {VatID} vatID * @param {DeliveryStatus} status - * @param {boolean} decrementReapCount + * @param {boolean} measureDirt * @param {MeterID} [meterID] + * @param {number} [gcKrefs] * @returns {CrankResults} */ - function deliveryCrankResults(vatID, status, decrementReapCount, meterID) { + function deliveryCrankResults(vatID, status, measureDirt, meterID, gcKrefs) { let meterUnderrun = false; let computrons; if (status.metering?.compute) { @@ -502,8 +504,16 @@ export default function buildKernel( results.terminate = { vatID, ...status.vatRequestedTermination }; } - if (decrementReapCount && !(results.abort || results.terminate)) { - results.decrementReapCount = { vatID }; + if (measureDirt && !(results.abort || results.terminate)) { + const dirt = { deliveries: 1 }; + if (computrons) { + // this is BigInt, but we use plain Number in Dirt records + dirt.computrons = Number(computrons); + } + if (gcKrefs) { + dirt.gcKrefs = gcKrefs; + } + results.measureDirt = { vatID, dirt }; } // We leave results.consumeMessage up to the caller. Send failures @@ -542,7 +552,8 @@ export default function buildKernel( } const status = await deliverAndLogToVat(vatID, kd, vd); - return deliveryCrankResults(vatID, status, true, meterID); + const gcKrefs = undefined; // TODO maybe increase by number of vrefs in args? + return deliveryCrankResults(vatID, status, true, meterID, gcKrefs); } /** @@ -588,7 +599,8 @@ export default function buildKernel( const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd); vatKeeper.deleteCListEntriesForKernelSlots(targets); const status = await deliverAndLogToVat(vatID, kd, vd); - return deliveryCrankResults(vatID, status, true, meterID); + const gcKrefs = undefined; // TODO maybe increase by number of vrefs in args? + return deliveryCrankResults(vatID, status, true, meterID, gcKrefs); } /** @@ -616,7 +628,9 @@ export default function buildKernel( } const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd); const status = await deliverAndLogToVat(vatID, kd, vd); - return deliveryCrankResults(vatID, status, false); // no meterID + const meterID = undefined; // no meterID + const gcKrefs = krefs.length; + return deliveryCrankResults(vatID, status, true, meterID, gcKrefs); } /** @@ -631,11 +645,14 @@ export default function buildKernel( if (!vatWarehouse.lookup(vatID)) { return NO_DELIVERY_CRANK_RESULTS; // can't collect from the dead } + const vatKeeper = kernelKeeper.provideVatKeeper(vatID); /** @type { KernelDeliveryBringOutYourDead } */ const kd = harden([type]); const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd); const status = await deliverAndLogToVat(vatID, kd, vd); - return deliveryCrankResults(vatID, status, false); // no meter + vatKeeper.clearReapDirt(); // BOYD zeros out the when-to-BOYD counters + // no gcKrefs, BOYD clears them anyways + return deliveryCrankResults(vatID, status, false); // no meter, BOYD clears dirt } /** @@ -676,8 +693,9 @@ export default function buildKernel( const status = await deliverAndLogToVat(vatID, kd, vd); // note: if deliveryCrankResults() learns to suspend vats, // startVat errors should still terminate them + const gcKrefs = undefined; // TODO maybe increase by number of vrefs in args? const results = harden({ - ...deliveryCrankResults(vatID, status, true, meterID), + ...deliveryCrankResults(vatID, status, true, meterID, gcKrefs), consumeMessage: true, }); return results; @@ -742,9 +760,17 @@ export default function buildKernel( function setKernelVatOption(vatID, option, value) { switch (option) { case 'reapInterval': { + // This still controls reapDirtThreshold.deliveries, and we do not + // yet offer controls for the other limits (gcKrefs or computrons). if (value === 'never' || isNat(value)) { const vatKeeper = kernelKeeper.provideVatKeeper(vatID); - vatKeeper.updateReapInterval(value); + const threshold = { ...vatKeeper.getReapDirtThreshold() }; + if (value === 'never') { + threshold.deliveries = value; + } else { + threshold.deliveries = Number(value); + } + vatKeeper.setReapDirtThreshold(threshold); } else { console.log(`WARNING: invalid reapInterval value`, value); } @@ -884,6 +910,7 @@ export default function buildKernel( const boydVD = vatWarehouse.kernelDeliveryToVatDelivery(vatID, boydKD); const boydStatus = await deliverAndLogToVat(vatID, boydKD, boydVD); const boydResults = deliveryCrankResults(vatID, boydStatus, false); + vatKeeper.clearReapDirt(); // we don't meter bringOutYourDead since no user code is running, but we // still report computrons to the runPolicy @@ -958,7 +985,14 @@ export default function buildKernel( startVatKD, startVatVD, ); - const startVatResults = deliveryCrankResults(vatID, startVatStatus, false); + const gcKrefs = undefined; // TODO maybe increase by number of vrefs in args? + const startVatResults = deliveryCrankResults( + vatID, + startVatStatus, + true, + meterID, + gcKrefs, + ); computrons = addComputrons(computrons, startVatResults.computrons); if (startVatResults.terminate) { @@ -1299,13 +1333,11 @@ export default function buildKernel( } } } - if (crankResults.decrementReapCount) { + if (crankResults.measureDirt) { // deliveries cause garbage, garbage needs collection - const { vatID } = crankResults.decrementReapCount; + const { vatID, dirt } = crankResults.measureDirt; const vatKeeper = kernelKeeper.provideVatKeeper(vatID); - if (vatKeeper.countdownToReap()) { - kernelKeeper.scheduleReap(vatID); - } + vatKeeper.addDirt(dirt); // might schedule a reap for that vat } // Vat termination (during delivery) is triggered by an illegal @@ -1579,10 +1611,14 @@ export default function buildKernel( 'bundleID', 'enablePipelining', 'reapInterval', + 'reapGCKrefs', + 'neverReap', ]); const { bundleID = 'b1-00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', reapInterval = 'never', + reapGCKrefs = 'never', + neverReap = false, enablePipelining, } = creationOptions; const vatID = kernelKeeper.allocateVatIDForNameIfNeeded(name); @@ -1594,6 +1630,8 @@ export default function buildKernel( const options = { name, reapInterval, + reapGCKrefs, + neverReap, enablePipelining, managerType, }; @@ -1740,14 +1778,36 @@ export default function buildKernel( } function changeKernelOptions(options) { - assertKnownOptions(options, ['defaultReapInterval', 'snapshotInterval']); + assertKnownOptions(options, [ + 'defaultReapInterval', + 'defaultReapGCKrefs', + 'snapshotInterval', + ]); kernelKeeper.startCrank(); try { for (const option of Object.getOwnPropertyNames(options)) { const value = options[option]; switch (option) { case 'defaultReapInterval': { - kernelKeeper.setDefaultReapInterval(value); + assert( + (typeof value === 'number' && value > 0) || value === 'never', + `defaultReapInterval ${value} must be a positive number or "never"`, + ); + kernelKeeper.setDefaultReapDirtThreshold({ + ...kernelKeeper.getDefaultReapDirtThreshold(), + deliveries: value, + }); + break; + } + case 'defaultReapGCKrefs': { + assert( + (typeof value === 'number' && value > 0) || value === 'never', + `defaultReapGCKrefs ${value} must be a positive number or "never"`, + ); + kernelKeeper.setDefaultReapDirtThreshold({ + ...kernelKeeper.getDefaultReapDirtThreshold(), + gcKrefs: value, + }); break; } case 'snapshotInterval': { diff --git a/packages/SwingSet/src/kernel/state/kernelKeeper.js b/packages/SwingSet/src/kernel/state/kernelKeeper.js index 2853333de8f..e5a1db20c41 100644 --- a/packages/SwingSet/src/kernel/state/kernelKeeper.js +++ b/packages/SwingSet/src/kernel/state/kernelKeeper.js @@ -1,6 +1,11 @@ +/* eslint-disable no-use-before-define */ import { Nat, isNat } from '@endo/nat'; import { assert, Fail } from '@endo/errors'; -import { initializeVatState, makeVatKeeper } from './vatKeeper.js'; +import { + initializeVatState, + makeVatKeeper, + DEFAULT_REAP_DIRT_THRESHOLD_KEY, +} from './vatKeeper.js'; import { initializeDeviceState, makeDeviceKeeper } from './deviceKeeper.js'; import { parseReachableAndVatSlot } from './reachable.js'; import { insistStorageAPI } from '../../lib/storageAPI.js'; @@ -33,14 +38,17 @@ const enableKernelGC = true; * @typedef { import('../../types-external.js').BundleCap } BundleCap * @typedef { import('../../types-external.js').BundleID } BundleID * @typedef { import('../../types-external.js').EndoZipBase64Bundle } EndoZipBase64Bundle - * @typedef { import('../../types-external.js').KernelOptions } KernelOptions * @typedef { import('../../types-external.js').KernelSlog } KernelSlog * @typedef { import('../../types-external.js').ManagerType } ManagerType * @typedef { import('../../types-external.js').SnapStore } SnapStore * @typedef { import('../../types-external.js').TranscriptStore } TranscriptStore * @typedef { import('../../types-external.js').VatKeeper } VatKeeper + * @typedef { import('../../types-internal.js').InternalKernelOptions } InternalKernelOptions + * @typedef { import('../../types-internal.js').ReapDirtThreshold } ReapDirtThreshold */ +export { DEFAULT_REAP_DIRT_THRESHOLD_KEY }; + // Kernel state lives in a key-value store supporting key retrieval by // lexicographic range. All keys and values are strings. // We simulate a tree by concatenating path-name components with ".". When we @@ -52,8 +60,9 @@ const enableKernelGC = true; // allowed to vary between instances in a consensus machine. Everything else // is required to be deterministic. // -// The schema is: +// The current ("v1") schema is: // +// version = '1' // vat.names = JSON([names..]) // vat.dynamicIDs = JSON([vatIDs..]) // vat.name.$NAME = $vatID = v$NN @@ -68,13 +77,22 @@ const enableKernelGC = true; // bundle.$BUNDLEID = JSON(bundle) // // kernel.defaultManagerType = managerType -// kernel.defaultReapInterval = $NN +// (old) kernel.defaultReapInterval = $NN +// kernel.defaultReapDirtThreshold = JSON({ thresholds }) +// thresholds (all optional) +// deliveries: number or 'never' (default) +// gcKrefs: number or 'never' (default) +// computrons: number or 'never' (default) +// never: boolean (default false) // kernel.relaxDurabilityRules = missing | 'true' // kernel.snapshotInitial = $NN // kernel.snapshotInterval = $NN // v$NN.source = JSON({ bundle }) or JSON({ bundleName }) -// v$NN.options = JSON +// v$NN.options = JSON , options include: +// .reapDirtThreshold = JSON({ thresholds }) +// thresholds (all optional, default to kernel-wide defaultReapDirtThreshold) +// (leave room for .snapshotDirtThreshold for #6786) // v$NN.o.nextID = $NN // v$NN.p.nextID = $NN // v$NN.d.nextID = $NN @@ -84,8 +102,10 @@ const enableKernelGC = true; // v$NN.c.$vatSlot = $kernelSlot = ko$NN/kp$NN/kd$NN // v$NN.vs.$key = string // v$NN.meter = m$NN // XXX does this exist? -// v$NN.reapInterval = $NN or 'never' -// v$NN.reapCountdown = $NN or 'never' +// old (v0): v$NN.reapInterval = $NN or 'never' +// old (v0): v$NN.reapCountdown = $NN or 'never' +// v$NN.reapDirt = JSON({ deliveries, gcKrefs, computrons }) // missing keys treated as zero +// (leave room for v$NN.snapshotDirt and options.snapshotDirtThreshold for #6786) // exclude from consensus // local.* @@ -132,6 +152,14 @@ const enableKernelGC = true; // Prefix reserved for host written data: // host. +// Kernel state schemas. The 'version' key records the state of the +// database, and is only modified by a call to upgradeSwingset(). +// +// v0: the original +// v1: replace `kernel.defaultReapInterval` with `kernel.defaultReapDirtThreshold` +// replace vat's `vNN.reapInterval`/`vNN.reapCountdown` with `vNN.reapDirt` +// and a `vNN.reapDirtThreshold` in `vNN.options` + export function commaSplit(s) { if (s === '') { return []; @@ -145,6 +173,21 @@ function insistMeterID(m) { Nat(BigInt(m.slice(1))); } +export const getAllStaticVats = kvStore => { + const result = []; + const prefix = 'vat.name.'; + for (const k of enumeratePrefixedKeys(kvStore, prefix)) { + const vatID = kvStore.get(k) || Fail`getNextKey ensures get`; + const name = k.slice(prefix.length); + result.push([name, vatID]); + } + return result; +}; + +export const getAllDynamicVats = getRequired => { + return JSON.parse(getRequired('vat.dynamicIDs')); +}; + // we use different starting index values for the various vNN/koNN/kdNN/kpNN // slots, to reduce confusing overlap when looking at debug messages (e.g. // seeing both kp1 and ko1, which are completely unrelated despite having the @@ -163,6 +206,23 @@ const FIRST_PROMISE_ID = 40n; const FIRST_CRANK_NUMBER = 0n; const FIRST_METER_ID = 1n; +// this default "reap interval" is low for the benefit of tests: +// applications should set it to something higher (perhaps 200) based +// on their expected usage + +export const DEFAULT_DELIVERIES_PER_BOYD = 1; + +// "20" will trigger a BOYD after 10 krefs are dropped and retired +// (drops and retires are delivered in separate messages, so +// 10+10=20). The worst case work-expansion we've seen is in #8401, +// where one drop breaks one cycle, and each cycle's cleanup causes 50 +// syscalls in the next v9-zoe BOYD. So this should limit each BOYD +// to cleaning 10 cycles, in 500 syscalls. + +export const DEFAULT_GC_KREFS_PER_BOYD = 20; + +const EXPECTED_VERSION = 1; + /** * @param {SwingStoreKernelStorage} kernelStorage * @param {KernelSlog|null} kernelSlog @@ -182,6 +242,16 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { return kvStore.get(key); } + if ( + !kvStore.has('version') || + Number(getRequired('version')) !== EXPECTED_VERSION + ) { + const have = kvStore.get('version') || 'undefined'; + throw Error( + `kernelStorage is too old (have ${have}, need ${EXPECTED_VERSION}), please upgradeSwingset()`, + ); + } + const { incStat, decStat, @@ -290,12 +360,17 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { } /** - * @param {KernelOptions} kernelOptions + * @param {InternalKernelOptions} kernelOptions */ function createStartingKernelState(kernelOptions) { + // this should probably be a standalone function, not a method const { defaultManagerType = 'local', - defaultReapInterval = 1, + defaultReapDirtThreshold = { + deliveries: DEFAULT_DELIVERIES_PER_BOYD, + gcKrefs: DEFAULT_GC_KREFS_PER_BOYD, + computrons: 'never', + }, relaxDurabilityRules = false, snapshotInitial = 3, snapshotInterval = 200, @@ -317,7 +392,10 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { initQueue('acceptanceQueue'); kvStore.set('crankNumber', `${FIRST_CRANK_NUMBER}`); kvStore.set('kernel.defaultManagerType', defaultManagerType); - kvStore.set('kernel.defaultReapInterval', `${defaultReapInterval}`); + kvStore.set( + DEFAULT_REAP_DIRT_THRESHOLD_KEY, + JSON.stringify(defaultReapDirtThreshold), + ); kvStore.set('kernel.snapshotInitial', `${snapshotInitial}`); kvStore.set('kernel.snapshotInterval', `${snapshotInterval}`); if (relaxDurabilityRules) { @@ -352,21 +430,25 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { /** * - * @returns {number | 'never'} + * @returns { ReapDirtThreshold } */ - function getDefaultReapInterval() { - const r = getRequired('kernel.defaultReapInterval'); - const ri = r === 'never' ? r : Number.parseInt(r, 10); - assert(ri === 'never' || typeof ri === 'number', `k.dri is '${ri}'`); - return ri; + function getDefaultReapDirtThreshold() { + return JSON.parse(getRequired(DEFAULT_REAP_DIRT_THRESHOLD_KEY)); } - function setDefaultReapInterval(interval) { - assert( - interval === 'never' || isNat(interval), - 'invalid defaultReapInterval value', - ); - kvStore.set('kernel.defaultReapInterval', `${interval}`); + /** + * @param { ReapDirtThreshold } threshold + */ + function setDefaultReapDirtThreshold(threshold) { + assert.typeof(threshold, 'object'); + assert(threshold); + for (const [key, value] of Object.entries(threshold)) { + assert( + (typeof value === 'number' && value > 0) || value === 'never', + `threshold[${key}] ${value} must be a positive number or "never"`, + ); + } + kvStore.set(DEFAULT_REAP_DIRT_THRESHOLD_KEY, JSON.stringify(threshold)); } function getNat(key) { @@ -764,7 +846,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { let idx = 0; for (const dataSlot of capdata.slots) { - // eslint-disable-next-line no-use-before-define incrementRefCount(dataSlot, `resolve|${kernelSlot}|s${idx}`); idx += 1; } @@ -787,7 +868,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { function cleanupAfterTerminatedVat(vatID) { insistVatID(vatID); - // eslint-disable-next-line no-use-before-define const vatKeeper = provideVatKeeper(vatID); const exportPrefix = `${vatID}.c.o+`; const importPrefix = `${vatID}.c.o-`; @@ -1102,15 +1182,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { kvStore.set(KEY, JSON.stringify(dynamicVatIDs)); } - function getStaticVats() { - const result = []; - for (const k of enumeratePrefixedKeys(kvStore, 'vat.name.')) { - const name = k.slice(9); - const vatID = kvStore.get(k) || Fail`getNextKey ensures get`; - result.push([name, vatID]); - } - return result; - } + const getStaticVats = () => getAllStaticVats(kvStore); function getDevices() { const result = []; @@ -1122,9 +1194,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { return result; } - function getDynamicVats() { - return JSON.parse(getRequired('vat.dynamicIDs')); - } + const getDynamicVats = () => getAllDynamicVats(getRequired); function allocateUpgradeID() { const nextID = Nat(BigInt(getRequired(`vat.nextUpgradeID`))); @@ -1262,7 +1332,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { if (reachable === 0) { const ownerVatID = ownerOfKernelObject(kref); if (ownerVatID) { - // eslint-disable-next-line no-use-before-define const vatKeeper = provideVatKeeper(ownerVatID); const isReachable = vatKeeper.getReachableFlag(kref); if (isReachable) { @@ -1316,6 +1385,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { incStat, decStat, getCrankNumber, + scheduleReap, snapStore, ); ephemeral.vatKeepers.set(vatID, vk); @@ -1536,9 +1606,10 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { setInitialized, createStartingKernelState, getDefaultManagerType, - getDefaultReapInterval, getRelaxDurabilityRules, - setDefaultReapInterval, + getDefaultReapDirtThreshold, + setDefaultReapDirtThreshold, + getSnapshotInitial, getSnapshotInterval, setSnapshotInterval, diff --git a/packages/SwingSet/src/kernel/state/vatKeeper.js b/packages/SwingSet/src/kernel/state/vatKeeper.js index 481cc06c01c..2ebe31dc384 100644 --- a/packages/SwingSet/src/kernel/state/vatKeeper.js +++ b/packages/SwingSet/src/kernel/state/vatKeeper.js @@ -1,8 +1,9 @@ /** * Kernel's keeper of persistent state for a vat. */ -import { Nat, isNat } from '@endo/nat'; +import { Nat } from '@endo/nat'; import { assert, q, Fail } from '@endo/errors'; +import { isObject } from '@endo/marshal'; import { parseKernelSlot } from '../parseKernelSlots.js'; import { makeVatSlot, parseVatSlot } from '../../lib/parseVatSlots.js'; import { insistVatID } from '../../lib/id.js'; @@ -18,7 +19,9 @@ import { enumeratePrefixedKeys } from './storageHelper.js'; * @typedef { import('../../types-external.js').SnapStore } SnapStore * @typedef { import('../../types-external.js').SourceOfBundle } SourceOfBundle * @typedef { import('../../types-external.js').TranscriptStore } TranscriptStore + * @typedef { import('../../types-internal.js').Dirt } Dirt * @typedef { import('../../types-internal.js').VatManager } VatManager + * @typedef { import('../../types-internal.js').ReapDirtThreshold } ReapDirtThreshold * @typedef { import('../../types-internal.js').RecordedVatOptions } RecordedVatOptions * @typedef { import('../../types-internal.js').TranscriptEntry } TranscriptEntry * @import {TranscriptDeliverySaveSnapshot} from '../../types-internal.js' @@ -32,13 +35,28 @@ const FIRST_OBJECT_ID = 50n; const FIRST_PROMISE_ID = 60n; const FIRST_DEVICE_ID = 70n; +// TODO: we export this from vatKeeper.js, and import it from +// kernelKeeper.js, because both files need it, and we want to avoid +// an import cycle (kernelKeeper imports other things from vatKeeper), +// but it really wants to live in kernelKeeper not vatKeeper +export const DEFAULT_REAP_DIRT_THRESHOLD_KEY = + 'kernel.defaultReapDirtThreshold'; + +const isBundleSource = source => { + return ( + isObject(source) && + (isObject(source.bundle) || + typeof source.bundleName === 'string' || + typeof source.bundleID === 'string') + ); +}; + /** * Establish a vat's state. * * @param {*} kvStore The key-value store in which the persistent state will be kept * @param {*} transcriptStore Accompanying transcript store * @param {string} vatID The vat ID string of the vat in question - * TODO: consider making this part of makeVatKeeper * @param {SourceOfBundle} source * @param {RecordedVatOptions} options */ @@ -49,20 +67,18 @@ export function initializeVatState( source, options, ) { - assert(options.workerOptions, `vat ${vatID} options missing workerOptions`); - assert(source); - assert('bundle' in source || 'bundleName' in source || 'bundleID' in source); - assert.typeof(options, 'object'); - const count = options.reapInterval; - assert(count === 'never' || isNat(count), `bad reapCountdown ${count}`); + assert(isBundleSource(source), `vat ${vatID} source has wrong shape`); + assert( + isObject(options) && isObject(options.workerOptions), + `vat ${vatID} options is missing workerOptions`, + ); kvStore.set(`${vatID}.o.nextID`, `${FIRST_OBJECT_ID}`); kvStore.set(`${vatID}.p.nextID`, `${FIRST_PROMISE_ID}`); kvStore.set(`${vatID}.d.nextID`, `${FIRST_DEVICE_ID}`); + kvStore.set(`${vatID}.reapDirt`, JSON.stringify({})); kvStore.set(`${vatID}.source`, JSON.stringify(source)); kvStore.set(`${vatID}.options`, JSON.stringify(options)); - kvStore.set(`${vatID}.reapInterval`, `${count}`); - kvStore.set(`${vatID}.reapCountdown`, `${count}`); transcriptStore.initTranscript(vatID); } @@ -87,6 +103,7 @@ export function initializeVatState( * @param {*} incStat * @param {*} decStat * @param {*} getCrankNumber + * @param {*} scheduleReap * @param {SnapStore} [snapStore] * returns an object to hold and access the kernel's state for the given vat */ @@ -107,10 +124,16 @@ export function makeVatKeeper( incStat, decStat, getCrankNumber, + scheduleReap, snapStore = undefined, ) { insistVatID(vatID); + // note: calling makeVatKeeper() does not change the DB. Any + // initialization or upgrade must be complete before it is + // called. Only the methods returned by makeVatKeeper() will change + // the DB. + function getRequired(key) { const value = kvStore.get(key); if (value === undefined) { @@ -119,6 +142,8 @@ export function makeVatKeeper( return value; } + const reapDirtKey = `${vatID}.reapDirt`; + /** * @param {SourceOfBundle} source * @param {RecordedVatOptions} options @@ -148,33 +173,74 @@ export function makeVatKeeper( return harden(options); } - function updateReapInterval(reapInterval) { - reapInterval === 'never' || - isNat(reapInterval) || - Fail`bad reapInterval ${reapInterval}`; - kvStore.set(`${vatID}.reapInterval`, `${reapInterval}`); - if (reapInterval === 'never') { - kvStore.set(`${vatID}.reapCountdown`, 'never'); - } - } + // This is named "addDirt" because it should increment all dirt + // counters (both for reap/BOYD and for heap snapshotting). We don't + // have `heapSnapshotDirt` yet, but when we do, it should get + // incremented here. - function countdownToReap() { - const rawCount = getRequired(`${vatID}.reapCountdown`); - if (rawCount === 'never') { - return false; - } else { - const count = Number.parseInt(rawCount, 10); - if (count === 1) { - kvStore.set( - `${vatID}.reapCountdown`, - getRequired(`${vatID}.reapInterval`), - ); - return true; - } else { - kvStore.set(`${vatID}.reapCountdown`, `${count - 1}`); - return false; + /** + * Add some "dirt" to the vat, possibly triggering a reap/BOYD. + * + * @param {Dirt} moreDirt + */ + function addDirt(moreDirt) { + assert.typeof(moreDirt, 'object'); + const reapDirt = JSON.parse(getRequired(reapDirtKey)); + const thresholds = { + ...JSON.parse(getRequired(DEFAULT_REAP_DIRT_THRESHOLD_KEY)), + ...JSON.parse(getRequired(`${vatID}.options`)).reapDirtThreshold, + }; + let reap = false; + for (const key of Object.keys(moreDirt)) { + const threshold = thresholds[key]; + // Don't accumulate dirt if it can't eventually trigger a + // BOYD. This is mainly to keep comms from counting upwards + // forever. TODO revisit this when we add heapSnapshotDirt, + // maybe check both thresholds and accumulate the dirt if either + // one is non-'never'. + if (threshold && threshold !== 'never') { + const oldDirt = reapDirt[key] || 0; + // The 'moreDirt' value might be Number or BigInt (eg + // .computrons). We coerce to Number so we can JSON-stringify. + const newDirt = oldDirt + Number(moreDirt[key]); + reapDirt[key] = newDirt; + if (newDirt >= threshold) { + reap = true; + } } } + if (!thresholds.never) { + kvStore.set(reapDirtKey, JSON.stringify(reapDirt)); + if (reap) { + scheduleReap(vatID); + } + } + } + + function getReapDirt() { + return JSON.parse(getRequired(reapDirtKey)); + } + + function clearReapDirt() { + // This is only called after a BOYD, so it should only clear the + // reap/BOYD counters. If/when we add heap-snapshot counters, + // those should get cleared in a separate clearHeapSnapshotDirt() + // function. + const reapDirt = {}; + kvStore.set(reapDirtKey, JSON.stringify(reapDirt)); + } + + function getReapDirtThreshold() { + return getOptions().reapDirtThreshold; + } + + /** + * @param {ReapDirtThreshold} reapDirtThreshold + */ + function setReapDirtThreshold(reapDirtThreshold) { + assert.typeof(reapDirtThreshold, 'object'); + const options = { ...getOptions(), reapDirtThreshold }; + kvStore.set(`${vatID}.options`, JSON.stringify(options)); } function nextDeliveryNum() { @@ -669,8 +735,11 @@ export function makeVatKeeper( setSourceAndOptions, getSourceAndOptions, getOptions, - countdownToReap, - updateReapInterval, + addDirt, + getReapDirt, + clearReapDirt, + getReapDirtThreshold, + setReapDirtThreshold, nextDeliveryNum, getIncarnationNumber, importsKernelSlot, diff --git a/packages/SwingSet/src/kernel/vat-loader/manager-factory.js b/packages/SwingSet/src/kernel/vat-loader/manager-factory.js index 2b8de555d9c..cb1b28c68c9 100644 --- a/packages/SwingSet/src/kernel/vat-loader/manager-factory.js +++ b/packages/SwingSet/src/kernel/vat-loader/manager-factory.js @@ -45,7 +45,6 @@ export function makeVatManagerFactory({ 'enableSetup', 'useTranscript', 'critical', - 'reapInterval', 'sourcedConsole', 'name', ]); diff --git a/packages/SwingSet/src/kernel/vat-loader/vat-loader.js b/packages/SwingSet/src/kernel/vat-loader/vat-loader.js index 34fc179af0d..c67fb97fa9c 100644 --- a/packages/SwingSet/src/kernel/vat-loader/vat-loader.js +++ b/packages/SwingSet/src/kernel/vat-loader/vat-loader.js @@ -26,7 +26,7 @@ export function makeVatLoader(stuff) { 'enablePipelining', 'useTranscript', 'critical', - 'reapInterval', + 'reapDirtThreshold', ]; /** diff --git a/packages/SwingSet/src/lib/recordVatOptions.js b/packages/SwingSet/src/lib/recordVatOptions.js index 658b5f9f713..f3c6c19c0a5 100644 --- a/packages/SwingSet/src/lib/recordVatOptions.js +++ b/packages/SwingSet/src/lib/recordVatOptions.js @@ -10,7 +10,9 @@ export const makeVatOptionRecorder = (kernelKeeper, bundleHandler) => { enablePipelining = false, enableDisavow = false, useTranscript = true, - reapInterval = kernelKeeper.getDefaultReapInterval(), + reapInterval, + reapGCKrefs, + neverReap = false, critical = false, meterID = undefined, managerType = kernelKeeper.getDefaultManagerType(), @@ -21,6 +23,17 @@ export const makeVatOptionRecorder = (kernelKeeper, bundleHandler) => { if (unused.length) { Fail`OptionRecorder: ${vatID} unused options ${unused.join(',')}`; } + const reapDirtThreshold = {}; + if (reapInterval !== undefined) { + reapDirtThreshold.deliveries = reapInterval; + } + if (reapGCKrefs !== undefined) { + reapDirtThreshold.gcKrefs = reapGCKrefs; + } + if (neverReap) { + reapDirtThreshold.never = true; + } + // TODO no computrons knob? const workerOptions = await makeWorkerOptions( managerType, bundleHandler, @@ -35,10 +48,11 @@ export const makeVatOptionRecorder = (kernelKeeper, bundleHandler) => { enablePipelining, enableDisavow, useTranscript, - reapInterval, + reapDirtThreshold, critical, meterID, }); + // want vNN.options to be in place before provideVatKeeper, so it can cache reapDirtThreshold in RAM, so: kernelKeeper.createVatState(vatID, source, vatOptions); }; diff --git a/packages/SwingSet/src/types-external.js b/packages/SwingSet/src/types-external.js index a62c411bd6b..1f1805b2884 100644 --- a/packages/SwingSet/src/types-external.js +++ b/packages/SwingSet/src/types-external.js @@ -28,14 +28,14 @@ export {}; */ /** - * @typedef {{ - * defaultManagerType?: ManagerType, - * defaultReapInterval?: number | 'never', - * relaxDurabilityRules?: boolean, - * snapshotInitial?: number, - * snapshotInterval?: number, - * pinBootstrapRoot?: boolean, - * }} KernelOptions + * @typedef {object} KernelOptions + * @property {ManagerType} [defaultManagerType] + * @property {number | 'never'} [defaultReapGCKrefs] + * @property {number | 'never'} [defaultReapInterval] + * @property {boolean} [relaxDurabilityRules] + * @property {number} [snapshotInitial] + * @property {number} [snapshotInterval] + * @property {boolean} [pinBootstrapRoot] */ /** @@ -167,6 +167,7 @@ export {}; * bundleName: string * }} BundleName * @typedef {(SourceSpec | BundleSpec | BundleRef | BundleName ) & { + * bundleID?: BundleID, * creationOptions?: Record, * parameters?: Record, * }} SwingSetConfigProperties @@ -292,13 +293,15 @@ export {}; * reconstructed via replay. If false, no such record is kept. * Defaults to true. * @property { number | 'never' } [reapInterval] - * The interval (measured in number of deliveries to the vat) - * after which the kernel will deliver the 'bringOutYourDead' - * directive to the vat. If the value is 'never', - * 'bringOutYourDead' will never be delivered and the vat will - * be responsible for internally managing (in a deterministic - * manner) any visible effects of garbage collection. Defaults - * to the kernel's configured 'defaultReapInterval' value. + * Trigger a bringOutYourDead after the vat has received + * this many deliveries. If the value is 'never', + * 'bringOutYourDead' will not be triggered by a delivery + * count (but might be triggered for other reasons). + * @property { number | 'never' } [reapGCKrefs] + * Trigger a bringOutYourDead when the vat has been given + * this many krefs in GC deliveries (dropImports, + * retireImports, retireExports). If the value is 'never', + * GC deliveries and their krefs are not treated specially. * @property { boolean } [critical] */ diff --git a/packages/SwingSet/src/types-internal.js b/packages/SwingSet/src/types-internal.js index 9140db5b5ec..102262e7db7 100644 --- a/packages/SwingSet/src/types-internal.js +++ b/packages/SwingSet/src/types-internal.js @@ -1,6 +1,19 @@ export {}; /** + * The host provides (external) KernelOptions as part of the + * SwingSetConfig record it passes to initializeSwingset(). This + * internal type represents the modified form passed to + * initializeKernel() and kernelKeeper.createStartingKernelState . + * + * @typedef {object} InternalKernelOptions + * @property {ManagerType} [defaultManagerType] + * @property {ReapDirtThreshold} [defaultReapDirtThreshold] + * @property {boolean} [relaxDurabilityRules] + * @property {number} [snapshotInitial] + * @property {number} [snapshotInterval] + * + * * The internal data that controls which worker we use (and how we use it) is * stored in a WorkerOptions record, which comes in "local", "node-subprocess", * and "xsnap" flavors. @@ -35,11 +48,48 @@ export {}; * @property { boolean } enableSetup * @property { boolean } enablePipelining * @property { boolean } useTranscript - * @property { number | 'never' } reapInterval + * @property { ReapDirtThreshold } reapDirtThreshold * @property { boolean } critical * @property { MeterID } [meterID] // property must be present, but can be undefined * @property { WorkerOptions } workerOptions * @property { boolean } enableDisavow + * + * @typedef ChangeVatOptions + * @property {number} [reapInterval] + */ + +/** + * Reap/BringOutYourDead/BOYD Scheduling + * + * We trigger a BringOutYourDead delivery (which "reaps" all dead + * objects from the vat) after a certain threshold of "dirt" has + * accumulated. This type is used to define the thresholds for three + * counters: 'deliveries', 'gcKrefs', and 'computrons'. If a property + * is a number, we trigger BOYD when the counter for that property + * exceeds the threshold value. If a property is the string 'never' or + * missing we do not use that counter to trigger BOYD. + * + * Each vat has a .reapDirtThreshold in their vNN.options record, + * which overrides the kernel-wide settings in + * 'kernel.defaultReapDirtThreshold' + * + * @typedef {object} ReapDirtThreshold + * @property {number | 'never'} [deliveries] + * @property {number | 'never'} [gcKrefs] + * @property {number | 'never'} [computrons] + * @property {boolean} [never] + */ + +/** + * Each counter in Dirt matches a threshold in + * ReapDirtThreshold. Missing values are treated as zero, so vats + * start with {} and accumulate dirt as deliveries are made, until a + * BOYD clears them. + * + * @typedef {object} Dirt + * @property {number} [deliveries] + * @property {number} [gcKrefs] + * @property {number} [computrons] */ /** @@ -86,7 +136,7 @@ export {}; * @typedef { { type: 'upgrade-vat', vatID: VatID, upgradeID: string, * bundleID: BundleID, vatParameters: SwingSetCapData, * upgradeMessage: string } } RunQueueEventUpgradeVat - * @typedef { { type: 'changeVatOptions', vatID: VatID, options: Record } } RunQueueEventChangeVatOptions + * @typedef { { type: 'changeVatOptions', vatID: VatID, options: ChangeVatOptions } } RunQueueEventChangeVatOptions * @typedef { { type: 'startVat', vatID: VatID, vatParameters: SwingSetCapData } } RunQueueEventStartVat * @typedef { { type: 'dropExports', vatID: VatID, krefs: string[] } } RunQueueEventDropExports * @typedef { { type: 'retireExports', vatID: VatID, krefs: string[] } } RunQueueEventRetireExports diff --git a/packages/SwingSet/test/bundling/bundles-kernel.test.js b/packages/SwingSet/test/bundling/bundles-kernel.test.js index cd6964e8682..8ec75bfb38a 100644 --- a/packages/SwingSet/test/bundling/bundles-kernel.test.js +++ b/packages/SwingSet/test/bundling/bundles-kernel.test.js @@ -12,7 +12,8 @@ import { initializeKernel } from '../../src/controller/initializeKernel.js'; test('install bundle', async t => { const endowments = makeKernelEndowments(); const { bundleStore } = endowments.kernelStorage; - await initializeKernel({}, endowments.kernelStorage); + const kconfig = { vats: {}, namedBundleIDs: {}, idToBundle: {} }; + await initializeKernel(kconfig, endowments.kernelStorage); const kernel = buildKernel(endowments, {}, {}); await kernel.start(); // empty queue diff --git a/packages/SwingSet/test/change-parameters/change-parameters.test.js b/packages/SwingSet/test/change-parameters/change-parameters.test.js index 0e2de993dee..200206f2f85 100644 --- a/packages/SwingSet/test/change-parameters/change-parameters.test.js +++ b/packages/SwingSet/test/change-parameters/change-parameters.test.js @@ -29,14 +29,22 @@ async function testChangeParameters(t) { t.teardown(c.shutdown); c.pinVatRoot('bootstrap'); await c.run(); - t.is(kvStore.get('kernel.defaultReapInterval'), '1'); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + deliveries: 1, + gcKrefs: 20, + computrons: 'never', + }); c.changeKernelOptions({ snapshotInterval: 1000, defaultReapInterval: 10, }); - t.is(kvStore.get('kernel.defaultReapInterval'), '10'); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + deliveries: 10, + gcKrefs: 20, + computrons: 'never', + }); t.throws(() => c.changeKernelOptions({ defaultReapInterval: 'banana' }), { - message: 'invalid defaultReapInterval value', + message: 'defaultReapInterval banana must be a positive number or "never"', }); t.throws(() => c.changeKernelOptions({ snapshotInterval: 'elephant' }), { message: 'invalid heap snapshotInterval value', @@ -44,6 +52,14 @@ async function testChangeParameters(t) { t.throws(() => c.changeKernelOptions({ baz: 'howdy' }), { message: 'unknown option "baz"', }); + c.changeKernelOptions({ + defaultReapGCKrefs: 77, + }); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + deliveries: 10, + gcKrefs: 77, + computrons: 'never', + }); async function run(method, args = []) { assert(Array.isArray(args)); @@ -57,7 +73,10 @@ async function testChangeParameters(t) { // setup target vat const [prepStatus] = await run('prepare', []); t.is(prepStatus, 'fulfilled'); - t.is(kvStore.get('v6.reapInterval'), '10'); + // the vat was created without option overrides, so + // reapDirtThreshold will be empty (everything defaults to the + // kernel-wide values) + t.deepEqual(JSON.parse(kvStore.get('v6.options')).reapDirtThreshold, {}); // now fiddle with stuff const [c1Status, c1Result] = await run('change', [{ foo: 47 }]); @@ -71,7 +90,9 @@ async function testChangeParameters(t) { const [c4Status, c4Result] = await run('change', [{ reapInterval: 20 }]); t.is(c4Status, 'fulfilled'); t.is(c4Result, 'ok'); - t.is(kvStore.get('v6.reapInterval'), '20'); + t.deepEqual(JSON.parse(kvStore.get('v6.options')).reapDirtThreshold, { + deliveries: 20, + }); } test('change vat options', async t => { diff --git a/packages/SwingSet/test/clist.test.js b/packages/SwingSet/test/clist.test.js index 10237c76322..df2be62545b 100644 --- a/packages/SwingSet/test/clist.test.js +++ b/packages/SwingSet/test/clist.test.js @@ -6,15 +6,18 @@ import { initSwingStore } from '@agoric/swing-store'; import { makeDummySlogger } from '../src/kernel/slogger.js'; import makeKernelKeeper from '../src/kernel/state/kernelKeeper.js'; +const CURRENT_VERSION = 1; + test(`clist reachability`, async t => { const slog = makeDummySlogger({}); const kernelStorage = initSwingStore(null).kernelStorage; + kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); const kk = makeKernelKeeper(kernelStorage, slog); const s = kk.kvStore; kk.createStartingKernelState({ defaultManagerType: 'local' }); const vatID = kk.allocateUnusedVatID(); const source = { bundleID: 'foo' }; - const options = { workerOptions: 'foo', reapInterval: 1 }; + const options = { workerOptions: {}, reapDirtThreshold: {} }; kk.createVatState(vatID, source, options); const vk = kk.provideVatKeeper(vatID); @@ -97,12 +100,13 @@ test(`clist reachability`, async t => { test('getImporters', async t => { const slog = makeDummySlogger({}); const kernelStorage = initSwingStore(null).kernelStorage; + kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); const kk = makeKernelKeeper(kernelStorage, slog); kk.createStartingKernelState({ defaultManagerType: 'local' }); const vatID1 = kk.allocateUnusedVatID(); const source = { bundleID: 'foo' }; - const options = { workerOptions: 'foo', reapInterval: 1 }; + const options = { workerOptions: {}, reapDirtThreshold: {} }; kk.createVatState(vatID1, source, options); kk.addDynamicVatID(vatID1); const vk1 = kk.provideVatKeeper(vatID1); diff --git a/packages/SwingSet/test/controller.test.js b/packages/SwingSet/test/controller.test.js index e731182d6fc..6994081b4ea 100644 --- a/packages/SwingSet/test/controller.test.js +++ b/packages/SwingSet/test/controller.test.js @@ -11,6 +11,7 @@ import { initializeSwingset, makeSwingsetController, } from '../src/index.js'; +import makeKernelKeeper from '../src/kernel/state/kernelKeeper.js'; import { checkKT } from './util.js'; const emptyVP = kser({}); @@ -487,3 +488,16 @@ test.serial('bootstrap export', async t => { removeTriple(kt, vattp0, vatTPVatID, 'o+0'); checkKT(t, c, kt); }); + +test('comms vat does not BOYD', async t => { + const config = {}; + const kernelStorage = initSwingStore().kernelStorage; + const controller = await buildVatController(config, [], { kernelStorage }); + t.teardown(controller.shutdown); + const k = makeKernelKeeper(kernelStorage, null); + const commsVatID = k.getVatIDForName('comms'); + t.deepEqual( + JSON.parse(k.kvStore.get(`${commsVatID}.options`)).reapDirtThreshold, + { never: true }, + ); +}); diff --git a/packages/SwingSet/test/kernel.test.js b/packages/SwingSet/test/kernel.test.js index 95dc619a19b..f18a0720ff3 100644 --- a/packages/SwingSet/test/kernel.test.js +++ b/packages/SwingSet/test/kernel.test.js @@ -3,7 +3,7 @@ import { test } from '../tools/prepare-test-env-ava.js'; import { Fail } from '@endo/errors'; -import { kser, kslot } from '@agoric/kmarshal'; +import { kser, kunser, kslot } from '@agoric/kmarshal'; import buildKernel from '../src/kernel/index.js'; import { initializeKernel } from '../src/controller/initializeKernel.js'; import { makeVatSlot } from '../src/lib/parseVatSlots.js'; @@ -1567,10 +1567,14 @@ test('xs-worker default manager type', async t => { ); }); -async function reapTest(t, freq) { - const kernel = await makeKernel(); +async function reapTest(t, freq, overrideNever) { + const endowments = makeKernelEndowments(); + await initializeKernel({}, endowments.kernelStorage); + const kernel = buildKernel(endowments, {}, {}); await kernel.start(); + const { kernelStorage } = endowments; const log = []; + function setup() { function dispatch(vatDeliverObject) { if (vatDeliverObject[0] === 'startVat') { @@ -1583,6 +1587,20 @@ async function reapTest(t, freq) { await kernel.createTestVat('vat1', setup, {}, { reapInterval: freq }); const vat1 = kernel.vatNameToID('vat1'); t.deepEqual(log, []); + const options = JSON.parse(kernelStorage.kvStore.get(`${vat1}.options`)); + t.deepEqual(options.reapDirtThreshold, { + deliveries: freq, + gcKrefs: 'never', // createTestVat minimizes BOYD + }); + + if (overrideNever) { + // when upgradeSwingset v0->v1 encounters a non-reaping vat (like + // comms), it sets the .options reapDirtThreshold to `{ never: + // true }`, so verify that this inhibits BOYD + options.reapDirtThreshold = { never: true }; + kernelStorage.kvStore.set(`${vat1}.options`, JSON.stringify(options)); + freq = 'never'; + } const vatRoot = kernel.addExport(vat1, 'o+1'); function deliverMessage(ordinal) { @@ -1602,11 +1620,28 @@ async function reapTest(t, freq) { return ['bringOutYourDead']; } - for (let i = 0; i < 100; i += 1) { - deliverMessage(i); - } + t.deepEqual(JSON.parse(kernelStorage.kvStore.get(`${vat1}.reapDirt`)), {}); + deliverMessage(0); // enqueues only t.deepEqual(log, []); await kernel.run(); + + // The first delivery increments dirt.deliveries . If freq=1 that + // will trigger an immediate BOYD and resets the counter, but for + // the slower-interval cases the counter will be left at 1. + + const expected1 = {}; + if (freq !== 'never' && freq > 1) { + expected1.deliveries = 1; + } + t.deepEqual( + JSON.parse(kernelStorage.kvStore.get(`${vat1}.reapDirt`)), + expected1, + ); + + for (let i = 1; i < 100; i += 1) { + deliverMessage(i); // enqueues only + } + await kernel.run(); for (let i = 0; i < 100; i += 1) { t.deepEqual(log.shift(), matchMsg(i)); if (freq !== 'never' && (i + 1) % freq === 0) { @@ -1635,3 +1670,150 @@ test('reap interval 17', async t => { test('reap interval never', async t => { await reapTest(t, 'never'); }); + +test('reap interval override never', async t => { + await reapTest(t, 5, true); +}); + +// Set up two vats, one to export vrefs, the other to import/drop +// them. The first will get a reapDirtThreshold.gcKrefs, and will log +// when the kernel sends it BOYD. + +async function reapGCKrefsTest(t, freq, overrideNever) { + const endowments = makeKernelEndowments(); + await initializeKernel({}, endowments.kernelStorage); + const kernel = buildKernel(endowments, {}, {}); + await kernel.start(); + const { kernelStorage } = endowments; + // note: worker=local, otherwise snapshotInitial/Interval would interfere + + let boyds = 0; + let rxGCkrefs = 0; + let lastExported = 2; + + // vat-under-test, export vrefs on request, watch for BOYDs + function setup1(syscall) { + function dispatch(vatDeliverObject) { + if (vatDeliverObject[0] === 'startVat') { + return; // skip startVat + } + if (vatDeliverObject[0] === 'message') { + // export vrefs, one per message + const target = vatDeliverObject[2].methargs.slots[0]; + const vref = `o+${lastExported}`; + lastExported += 1; + syscall.send(target, kser(['hold', [kslot(vref)]])); + return; + } + if (vatDeliverObject[0] === 'bringOutYourDead') { + boyds += 1; + } + if (vatDeliverObject[0] === 'dropExports') { + rxGCkrefs += vatDeliverObject[1].length; + } + if (vatDeliverObject[0] === 'retireExports') { + rxGCkrefs += vatDeliverObject[1].length; + } + if (vatDeliverObject[0] === 'retireImports') { + rxGCkrefs += vatDeliverObject[1].length; + } + } + return dispatch; + } + const vat1 = await kernel.createTestVat( + 'vat1', + setup1, + {}, + { reapInterval: 'never', reapGCKrefs: freq }, + ); + const v1root = kernel.getRootObject(vat1); + kernel.pinObject(v1root); + + if (overrideNever) { + // when upgradeSwingset v0->v1 encounters a non-reaping vat (like + // comms), it sets the .options reapDirtThreshold to `{ never: + // true }`, so verify that this inhibits BOYD. It is especially + // important that this works against gcKrefs, otherwise we'd be + // BOYDing vat-comms all the time, which is pointless. + const options = JSON.parse(kernelStorage.kvStore.get(`${vat1}.options`)); + options.reapDirtThreshold = { never: true }; + kernelStorage.kvStore.set(`${vat1}.options`, JSON.stringify(options)); + freq = 'never'; + } + + // helper vat, imports vrefs, drops on request + function setup2(syscall) { + const hold = []; + function dispatch(vatDeliverObject) { + if (vatDeliverObject[0] === 'startVat') { + return; // skip startVat + } + if (vatDeliverObject[0] === 'message') { + const [meth, args] = kunser(vatDeliverObject[2].methargs); + if (meth === 'hold') { + for (const vref of vatDeliverObject[2].methargs.slots) { + hold.push(vref); + } + } else { + const [count] = args; + syscall.dropImports(hold.slice(0, count)); + syscall.retireImports(hold.slice(0, count)); + hold.splice(0, count); + } + } + } + return dispatch; + } + const vat2 = await kernel.createTestVat('vat2', setup2, {}); + const v2root = kernel.getRootObject(vat2); + kernel.pinObject(v2root); + + await kernel.run(); + t.is(boyds, 0); + + async function addExport() { + kernel.queueToKref(v1root, `pleaseExport`, [kslot(v2root)], 'none'); + await kernel.run(); + } + + async function doDrop(count) { + kernel.queueToKref(v2root, `drop`, [count], 'none'); + await kernel.run(); + } + + await addExport(); + await addExport(); + t.is(boyds, 0); + // c-list should currently have two krefs exported by the vat + + // now we drop one for every new one we add, and every 'interval'/2 + // we should see a BOYD + + let krefs = 0; + for (let i = 0; i < 10; i += 1) { + await addExport(); + await doDrop(1); + krefs += 2; + t.is(rxGCkrefs, krefs); + if (freq === 'never' || krefs < freq) { + t.is(boyds, 0); + } else { + t.is(boyds, 1); + boyds = 0; + krefs = 0; + rxGCkrefs = 0; + } + } +} + +test('reap gc-krefs 10', async t => { + await reapGCKrefsTest(t, 10); +}); + +test('reap gc-krefs 12', async t => { + await reapGCKrefsTest(t, 12); +}); + +test('reap gc-krefs overrideNever', async t => { + await reapGCKrefsTest(t, 12, true); +}); diff --git a/packages/SwingSet/test/snapshots/state.test.js.md b/packages/SwingSet/test/snapshots/state.test.js.md index 3ab81fd5cd5..5a7cae18e2a 100644 --- a/packages/SwingSet/test/snapshots/state.test.js.md +++ b/packages/SwingSet/test/snapshots/state.test.js.md @@ -8,8 +8,8 @@ Generated by [AVA](https://avajs.dev). > initial state - 'a5d302e6743578ccda03ea386abd49de0a3bf4d7dedda2f69585c663806c30bc' + '2cc47b69a725bb4a2bfca1e2ba2b8625e3a62261acac60e37be95ebc09b1e02e' > expected activityhash - 'f5f1f643f6242a73c79b0437dbab222d34642ea5d047f15aaf5551d5903711d3' + 'c7edd8883ba896276247c1de6391d1cdac3fcc6bfbd1599098dbd367e454b41f' diff --git a/packages/SwingSet/test/snapshots/state.test.js.snap b/packages/SwingSet/test/snapshots/state.test.js.snap index 632a77941e6c1d9757416e97e33d0d9d0ca7357c..0a7a22b266fec1f45e928a1857a0de7f00ee0290 100644 GIT binary patch literal 276 zcmV+v0qg!jRzV7L*p4dF`mug=oPe74e_3nA@kZF7P{K{f zN!f@BiK#}@d^7W%u&i`!pI7O*7r8vrE7aoxZ5a<^uDNF@W9!gmgvui@Z_!#?ZWYdL zS$q4sGDC-#=eyy4xM{Z^9(GTf$a-`Uyq*y)XiNl3G;S_9J&OUSwNpSqZW7~B;w(s0 zL?x3JDS%Q=e=7V>A}`D#1cMNw)81+8NwGK+6)R#OlZYH9WHrY)#lXnMd9kxatI{v& aO)h7lF55=x`91fK+X+7>P-~hX0RRAo7b=y(L@=6+UD>~ z3(<}@JRge)00000000ARkUdTVF%U*KLWr6hxT8Yydi=B7aRO@E&z`YjWwRUM0fiE7 zLQaYzDkRY-uUHRz^u2jBZEf1Hn66MUSlNKNY1`J``ifKE zZ`!{u%uu{M-wb!dmEG-dzk5PpvQA?16w*`*!;*leh|GRwv7lT%Cl+S78s{*D5+hAE z7P7g>f-KANPs0DE8tPc1ry8AiNL13Ct*68p**VALqj!S9)~7lKpoS1e4zr~+j!eI% aH#wiAsIWoN_TKd4I^hSg1)BXJ0RR94fPa+$ diff --git a/packages/SwingSet/test/state.test.js b/packages/SwingSet/test/state.test.js index d1ab7b83d50..3b37d83b502 100644 --- a/packages/SwingSet/test/state.test.js +++ b/packages/SwingSet/test/state.test.js @@ -7,6 +7,7 @@ import { createHash } from 'crypto'; import { kser, kslot } from '@agoric/kmarshal'; import { initSwingStore } from '@agoric/swing-store'; import makeKernelKeeper from '../src/kernel/state/kernelKeeper.js'; +import { upgradeSwingset } from '../src/controller/upgradeSwingset.js'; import { makeKernelStats } from '../src/kernel/state/stats.js'; import { KERNEL_STATS_METRICS } from '../src/kernel/metrics.js'; import { @@ -143,8 +144,11 @@ test('storage helpers', t => { ]); }); +const CURRENT_VERSION = 1; + function buildKeeperStorageInMemory() { const { kernelStorage, debug } = initSwingStore(null); + kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); return { ...debug, // serialize, dump ...kernelStorage, @@ -181,6 +185,7 @@ test('kernel state', async t => { k.emitCrankHashes(); checkState(t, store.dump, [ + ['version', '1'], ['crankNumber', '0'], ['initialized', 'true'], ['gcActions', '[]'], @@ -197,7 +202,10 @@ test('kernel state', async t => { ['kd.nextID', '30'], ['kp.nextID', '40'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -216,6 +224,7 @@ test('kernelKeeper vat names', async t => { k.emitCrankHashes(); checkState(t, store.dump, [ + ['version', '1'], ['crankNumber', '0'], ['gcActions', '[]'], ['runQueue', '[1,1]'], @@ -233,7 +242,10 @@ test('kernelKeeper vat names', async t => { ['vat.name.vatname5', 'v1'], ['vat.name.Frank', 'v2'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -266,6 +278,7 @@ test('kernelKeeper device names', async t => { k.emitCrankHashes(); checkState(t, store.dump, [ + ['version', '1'], ['crankNumber', '0'], ['gcActions', '[]'], ['runQueue', '[1,1]'], @@ -283,7 +296,10 @@ test('kernelKeeper device names', async t => { ['device.name.devicename5', 'd7'], ['device.name.Frank', 'd8'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -442,6 +458,7 @@ test('kernelKeeper promises', async t => { k.emitCrankHashes(); checkState(t, store.dump, [ + ['version', '1'], ['crankNumber', '0'], ['device.nextID', '7'], ['vat.nextID', '1'], @@ -465,7 +482,10 @@ test('kernelKeeper promises', async t => { [`${ko}.owner`, 'v1'], [`${ko}.refCount`, '1,1'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -513,7 +533,7 @@ test('vatKeeper', async t => { k.createStartingKernelState({ defaultManagerType: 'local' }); const v1 = k.allocateVatIDForNameIfNeeded('name1'); const source = { bundleID: 'foo' }; - const options = { workerOptions: 'foo', reapInterval: 1 }; + const options = { workerOptions: {}, reapDirtThreshold: {} }; k.createVatState(v1, source, options); const vk = k.provideVatKeeper(v1); @@ -555,7 +575,7 @@ test('vatKeeper.getOptions', async t => { 'b1-00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'; const source = { bundleID }; const workerOptions = { type: 'local' }; - const options = { workerOptions, name: 'fred', reapInterval: 1 }; + const options = { workerOptions, name: 'fred', reapDirtThreshold: {} }; k.createVatState(v1, source, options); const vk = k.provideVatKeeper(v1); @@ -925,3 +945,201 @@ test('stats - can load and save existing stats', t => { t.deepEqual(JSON.parse(getSerializedStats().consensusStats), consensusStats); t.deepEqual(JSON.parse(getSerializedStats().localStats), localStats); }); + +test('vatKeeper dirt counters', async t => { + const store = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(store, null); + k.createStartingKernelState({ + defaultManagerType: 'local', + }); + k.saveStats(); + + // the defaults are designed for testing + t.deepEqual(JSON.parse(k.kvStore.get(`kernel.defaultReapDirtThreshold`)), { + deliveries: 1, + gcKrefs: 20, + computrons: 'never', + }); + + const reapDirtThreshold = { deliveries: 10, gcKrefs: 20, computrons: 100 }; + const never = { deliveries: 'never', gcKrefs: 'never', computrons: 'never' }; + + // a new DB will have empty dirt entries for each vat created + const source = { bundleID: 'foo' }; + const v1 = k.allocateVatIDForNameIfNeeded('name1'); + k.createVatState(v1, source, { workerOptions: {}, reapDirtThreshold }); + const vk1 = k.provideVatKeeper(v1); + + const v2 = k.allocateVatIDForNameIfNeeded('name2'); + k.createVatState(v2, source, { workerOptions: {}, reapDirtThreshold }); + const vk2 = k.provideVatKeeper(v2); + + const v3 = k.allocateVatIDForNameIfNeeded('name3'); + k.createVatState(v3, source, { + workerOptions: {}, + reapDirtThreshold: never, + }); + const vk3 = k.provideVatKeeper(v3); + + // the nominal "all clean" entry is { deliveries: 0, gcKrefs: 0, + // computrons: 0 }, but we only store the non-zero keys, so it's + // really {} + t.deepEqual(vk1.getReapDirt(), {}); + t.deepEqual(vk2.getReapDirt(), {}); + + // our write-through cache should store the initial value in the DB + t.true(store.kvStore.has(`${v1}.reapDirt`)); + t.deepEqual(JSON.parse(store.kvStore.get(`${v1}.reapDirt`)), {}); + + // changing one entry doesn't change any others + vk1.addDirt({ deliveries: 1, gcKrefs: 0, computrons: 12 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 1, gcKrefs: 0, computrons: 12 }); + t.deepEqual(vk2.getReapDirt(), {}); + t.not(vk1.getReapDirt(), vk2.getReapDirt()); + // and writes through the cache + t.deepEqual(JSON.parse(store.kvStore.get(`${v1}.reapDirt`)), { + deliveries: 1, + gcKrefs: 0, + computrons: 12, + }); + + // clearing the dirt will zero out the entries + vk1.clearReapDirt(); + t.deepEqual(vk1.getReapDirt(), {}); + t.deepEqual(JSON.parse(store.kvStore.get(`${v1}.reapDirt`)), {}); + + // nothing has reached the threshold yet + t.is(k.nextReapAction(), undefined); + + vk1.addDirt({ deliveries: 4 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 4 }); + t.is(k.nextReapAction(), undefined); + vk1.addDirt({ deliveries: 5 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 9 }); + t.is(k.nextReapAction(), undefined); + vk1.addDirt({ deliveries: 6 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 15 }); + t.deepEqual(k.nextReapAction(), { type: 'bringOutYourDead', vatID: v1 }); + t.is(k.nextReapAction(), undefined); + + // dirt is ignored when the threshold is 'never' + vk3.addDirt({ deliveries: 4 }); + t.deepEqual(vk3.getReapDirt(), {}); +}); + +test('dirt upgrade', async t => { + const store = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(store, null); + k.createStartingKernelState({ + defaultManagerType: 'local', + }); + k.saveStats(); + const v1 = k.allocateVatIDForNameIfNeeded('name1'); + const source = { bundleID: 'foo' }; + // actual vats get options.reapDirtThreshold ; we install + // options.reapInterval to simulate the old version, and we use + // nonsense values because .reapInterval was not updated by + // changeVatOptions so the upgrade process should ignore it + const options = { workerOptions: {}, reapInterval: 666 }; + k.createVatState(v1, source, options); + // "v2" is like v1 but with the default reapInterval + const v2 = k.allocateVatIDForNameIfNeeded('name2'); + const options2 = { ...options, reapInterval: 667 }; + k.createVatState(v2, source, options2); + // "v3" is like comms: no BOYD + const v3 = k.allocateVatIDForNameIfNeeded('comms'); + const options3 = { ...options, reapInterval: 'never' }; + k.createVatState(v3, source, options3); + + // Test that upgrade from an older version of the DB will populate + // the right keys. We simulate the old version by modifying a + // serialized copy. The old version (on mainnet) had things like: + // * kernel.defaultReapInterval: 1000 + // * v1.options: { ... reapInterval: 1000 } + // * v1.reapCountdown: 123 + // * v1.reapInterval: 1000 + // * v2.options: { ... reapInterval: 300 } + // * v2.reapCountdown: 123 + // * v2.reapInterval: 300 + // * v3.options: { ... reapInterval: 'never' } + // * v3.reapCountdown: 'never' + // * v3.reapInterval: 'never' + + t.is(k.kvStore.get('version'), '1'); + k.kvStore.delete(`kernel.defaultReapDirtThreshold`); + k.kvStore.set(`kernel.defaultReapInterval`, '1000'); + + // v1 uses the default reapInterval + k.kvStore.delete(`${v1}.reapDirt`); + k.kvStore.delete(`${v1}.reapDirtThreshold`); + k.kvStore.set(`${v1}.reapInterval`, '1000'); + k.kvStore.set(`${v1}.reapCountdown`, '700'); + + // v2 uses a custom reapCountdown + k.kvStore.delete(`${v2}.reapDirt`); + k.kvStore.delete(`${v2}.reapDirtThreshold`); + k.kvStore.set(`${v2}.reapInterval`, '300'); + k.kvStore.set(`${v2}.reapCountdown`, '70'); + + // v3 is like comms and never reaps + k.kvStore.delete(`${v3}.reapDirt`); + k.kvStore.delete(`${v3}.reapDirtThreshold`); + k.kvStore.set(`${v3}.reapInterval`, 'never'); + k.kvStore.set(`${v3}.reapCountdown`, 'never'); + + k.kvStore.delete(`version`); + + // kernelKeeper refuses to work with an old state + t.throws(() => duplicateKeeper(store.serialize)); + + // it requires a manual upgrade + let k2; + { + const serialized = store.serialize(); + const { kernelStorage } = initSwingStore(null, { serialized }); + upgradeSwingset(kernelStorage); + k2 = makeKernelKeeper(kernelStorage, null); // works this time + k2.loadStats(); + } + + t.true(k2.kvStore.has(`kernel.defaultReapDirtThreshold`)); + // threshold.deliveries is converted from defaultReapInterval + t.deepEqual(JSON.parse(k2.kvStore.get(`kernel.defaultReapDirtThreshold`)), { + deliveries: 1000, + gcKrefs: 20, + computrons: 'never', + }); + + t.true(k2.kvStore.has(`${v1}.reapDirt`)); + // reapDirt.deliveries computed from old reapInterval-reapCountdown + t.deepEqual(JSON.parse(k2.kvStore.get(`${v1}.reapDirt`)), { + deliveries: 300, + }); + // reapDirtThreshold.deliveries computed from old reapInterval, and + // because it matches the kernel-wide default, the .options record + // is left empty + t.deepEqual( + JSON.parse(k2.kvStore.get(`${v1}.options`)).reapDirtThreshold, + {}, + ); + const vk1New = k2.provideVatKeeper(v1); + t.deepEqual(vk1New.getReapDirt(), { deliveries: 300 }); + + // v2 reapDirt is computed the same way + t.true(k2.kvStore.has(`${v2}.reapDirt`)); + t.deepEqual(JSON.parse(k2.kvStore.get(`${v2}.reapDirt`)), { + deliveries: 230, + }); + // the custom reapInterval is transformed into an .options override + t.deepEqual(JSON.parse(k2.kvStore.get(`${v2}.options`)).reapDirtThreshold, { + deliveries: 300, + }); + const vk2New = k2.provideVatKeeper(v2); + t.deepEqual(vk2New.getReapDirt(), { deliveries: 230 }); + + t.true(k2.kvStore.has(`${v3}.reapDirt`)); + t.deepEqual(JSON.parse(k2.kvStore.get(`${v3}.reapDirt`)), {}); + t.deepEqual(JSON.parse(k2.kvStore.get(`${v3}.options`)).reapDirtThreshold, { + never: true, + }); +}); diff --git a/packages/SwingSet/test/upgrade-swingset.test.js b/packages/SwingSet/test/upgrade-swingset.test.js new file mode 100644 index 00000000000..1a468543bf5 --- /dev/null +++ b/packages/SwingSet/test/upgrade-swingset.test.js @@ -0,0 +1,202 @@ +/* eslint-disable no-underscore-dangle */ +// @ts-nocheck + +import { initSwingStore } from '@agoric/swing-store'; +import { test } from '../tools/prepare-test-env-ava.js'; + +import { + initializeSwingset, + makeSwingsetController, + upgradeSwingset, + buildKernelBundles, +} from '../src/index.js'; + +test.before(async t => { + const kernelBundles = await buildKernelBundles(); + t.context.data = { kernelBundles }; +}); + +test('kernel refuses to run with out-of-date DB', async t => { + const { hostStorage, kernelStorage } = initSwingStore(); + const { commit } = hostStorage; + const { kvStore } = kernelStorage; + const config = {}; + await initializeSwingset(config, [], kernelStorage, t.context.data); + await commit(); + + // now doctor the initial state to make it look like the + // kernelkeeper v0 schema, just deleting the version key + + t.is(kvStore.get('version'), '1'); + kvStore.delete(`version`); + await commit(); + + // Now build a controller around this modified state, which should fail. + await t.throwsAsync(() => makeSwingsetController(kernelStorage), { + message: /kernelStorage is too old/, + }); +}); + +test('upgrade kernel state', async t => { + const { hostStorage, kernelStorage } = initSwingStore(); + const { commit } = hostStorage; + const { kvStore } = kernelStorage; + const config = { + vats: { + one: { + sourceSpec: new URL( + 'files-vattp/bootstrap-test-vattp.js', + import.meta.url, + ).pathname, + }, + }, + }; + await initializeSwingset(config, [], kernelStorage, t.context.data); + await commit(); + + // now doctor the initial state to make it look like the + // kernelkeeper v0 schema, with 'kernel.defaultReapInterval' instead + // of 'kernel.defaultReapDirtThreshold', and + // 'v1.reapCountdown`/`.reapInterval` . This is cribbed from "dirt + // upgrade" in test-state.js. + // + // our mainnet vats have data like: + // v5.options|{"workerOptions":{"type":"xsnap","bundleIDs":["b0-5c790a966210b78de758fb442af542714ed96da09db76e0b31d6a237e555fd62","b0-e0d2dafc7e981947b42118e8c950837109683bae56f7b4f5bffa1b67e5c1e768"]},"name":"timer","enableSetup":false,"enablePipelining":false,"enableDisavow":false,"useTranscript":true,"reapInterval":1000,"critical":false} + // v5.reapCountdown|181 + // v5.reapInterval|1000 + // + // This is a bit fragile.. there are probably ways to refactor + // kernelKeeper to make this better, or at least put all the + // manipulation/simulation code in the same place. + + t.true(kvStore.has('kernel.defaultReapDirtThreshold')); + + t.is(kvStore.get('version'), '1'); + kvStore.delete('version'); // i.e. revert to v0 + kvStore.delete(`kernel.defaultReapDirtThreshold`); + kvStore.set(`kernel.defaultReapInterval`, '300'); + + const vatIDs = {}; + for (const name of JSON.parse(kvStore.get('vat.names'))) { + const vatID = kvStore.get(`vat.name.${name}`); + t.truthy(vatID, name); + vatIDs[name] = vatID; + t.true(kvStore.has(`${vatID}.reapDirt`)); + kvStore.delete(`${vatID}.reapDirt`); + const options = JSON.parse(kvStore.get(`${vatID}.options`)); + t.truthy(options); + t.truthy(options.reapDirtThreshold); + delete options.reapDirtThreshold; + options.reapInterval = 55; // ignored by upgrader, so make it bogus + kvStore.set(`${vatID}.options`, JSON.stringify(options)); + if (name === 'comms') { + kvStore.set(`${vatID}.reapInterval`, 'never'); + kvStore.set(`${vatID}.reapCountdown`, 'never'); + } else { + kvStore.set(`${vatID}.reapInterval`, '100'); + kvStore.set(`${vatID}.reapCountdown`, '70'); + // 100-70 means the new state's dirt.deliveries should be 30 + } + } + + await commit(); + + // confirm that this state is too old for the kernel to use + await t.throwsAsync(() => makeSwingsetController(kernelStorage), { + message: /kernelStorage is too old/, + }); + + // upgrade it + upgradeSwingset(kernelStorage); + + // now we should be good to go + const _controller = await makeSwingsetController(kernelStorage); + + t.true(kvStore.has('kernel.defaultReapDirtThreshold')); + // the kernel-wide threshold gets a .gcKrefs (to meet our upcoming + // slow-deletion goals) + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + computrons: 'never', + deliveries: 300, + gcKrefs: 20, + }); + + // normal vat has some (computed) accumulated dirt + t.deepEqual(JSON.parse(kvStore.get(`${vatIDs.one}.reapDirt`)), { + deliveries: 30, + }); + // anywhere the vat's upgraded threshold differs from the + // kernel-wide threshold, .options gets an override value, in this + // case on deliveries (since 100 !== 300) + t.deepEqual( + JSON.parse(kvStore.get(`${vatIDs.one}.options`)).reapDirtThreshold, + { deliveries: 100 }, + ); + + // comms doesn't reap, and doesn't count dirt, and gets a special + // 'never' marker + t.deepEqual(JSON.parse(kvStore.get(`${vatIDs.comms}.reapDirt`)), {}); + t.deepEqual( + JSON.parse(kvStore.get(`${vatIDs.comms}.options`)).reapDirtThreshold, + { never: true }, + ); + + // TODO examine the state, use it + + // TODO check the export-data callbacks +}); + +test('upgrade non-reaping kernel state', async t => { + const { hostStorage, kernelStorage } = initSwingStore(); + const { commit } = hostStorage; + const { kvStore } = kernelStorage; + const config = {}; + await initializeSwingset(config, [], kernelStorage, t.context.data); + await commit(); + + // now doctor the initial state to make it look like the + // kernelkeeper v0 schema, with 'kernel.defaultReapInterval' of 'never' + + t.true(kvStore.has('kernel.defaultReapDirtThreshold')); + + t.is(kvStore.get('version'), '1'); + kvStore.delete('version'); // i.e. revert to v0 + kvStore.delete(`kernel.defaultReapDirtThreshold`); + kvStore.set(`kernel.defaultReapInterval`, 'never'); + + const vatIDs = {}; + for (const name of JSON.parse(kvStore.get('vat.names'))) { + const vatID = kvStore.get(`vat.name.${name}`); + t.truthy(vatID, name); + vatIDs[name] = vatID; + t.true(kvStore.has(`${vatID}.reapDirt`)); + kvStore.delete(`${vatID}.reapDirt`); + const options = JSON.parse(kvStore.get(`${vatID}.options`)); + t.truthy(options); + t.truthy(options.reapDirtThreshold); + delete options.reapDirtThreshold; + options.reapInterval = 'never'; + kvStore.set(`${vatID}.options`, JSON.stringify(options)); + kvStore.set(`${vatID}.reapInterval`, 'never'); + kvStore.set(`${vatID}.reapCountdown`, 'never'); + } + await commit(); + + // confirm that this state is too old for the kernel to use + await t.throwsAsync(() => makeSwingsetController(kernelStorage), { + message: /kernelStorage is too old/, + }); + + // upgrade it + upgradeSwingset(kernelStorage); + + // now we should be good to go + const _controller = await makeSwingsetController(kernelStorage); + + t.true(kvStore.has('kernel.defaultReapDirtThreshold')); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + computrons: 'never', + deliveries: 'never', + gcKrefs: 'never', + }); +}); diff --git a/packages/SwingSet/test/vat-admin/bootstrap.js b/packages/SwingSet/test/vat-admin/bootstrap.js index 5b1bcaa15ee..74bf050d9ac 100644 --- a/packages/SwingSet/test/vat-admin/bootstrap.js +++ b/packages/SwingSet/test/vat-admin/bootstrap.js @@ -25,6 +25,14 @@ export function buildRootObject() { return n; }, + async byNameWithOptions(bundleName, opts) { + const { root } = await E(admin).createVatByName(bundleName, { + ...options, + ...opts, + }); + return root; + }, + async byNamedBundleCap(name) { const bcap = await E(admin).getNamedBundleCap(name); const { root } = await E(admin).createVat(bcap, options); diff --git a/packages/SwingSet/test/vat-admin/create-vat.test.js b/packages/SwingSet/test/vat-admin/create-vat.test.js index a23a686752e..40b3325cc58 100644 --- a/packages/SwingSet/test/vat-admin/create-vat.test.js +++ b/packages/SwingSet/test/vat-admin/create-vat.test.js @@ -444,3 +444,40 @@ test('createVat holds refcount', async t => { await stepUntil(() => false); t.deepEqual(kunser(c.kpResolution(kpid)), 0); }); + +test('createVat without options', async t => { + const printSlog = false; + const { c, kernelStorage } = await doTestSetup(t, false, printSlog); + const { kvStore } = kernelStorage; + const threshold = JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')); + t.deepEqual(threshold, { deliveries: 1, gcKrefs: 20, computrons: 'never' }); + + const kpid = c.queueToVatRoot('bootstrap', 'byNameWithOptions', [ + 'new13', + {}, + ]); + await c.run(); + const kref = kunser(c.kpResolution(kpid)).getKref(); + const vatID = kvStore.get(`${kref}.owner`); + const options = JSON.parse(kvStore.get(`${vatID}.options`)); + t.deepEqual(options.reapDirtThreshold, {}); +}); + +test('createVat with options', async t => { + const printSlog = false; + const { c, kernelStorage } = await doTestSetup(t, false, printSlog); + const { kvStore } = kernelStorage; + const threshold = JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')); + t.deepEqual(threshold, { deliveries: 1, gcKrefs: 20, computrons: 'never' }); + + const opts = { reapInterval: 123 }; + const kpid = c.queueToVatRoot('bootstrap', 'byNameWithOptions', [ + 'new13', + opts, + ]); + await c.run(); + const kref = kunser(c.kpResolution(kpid)).getKref(); + const vatID = kvStore.get(`${kref}.owner`); + const options = JSON.parse(kvStore.get(`${vatID}.options`)); + t.deepEqual(options.reapDirtThreshold, { deliveries: 123 }); +}); From ff1e9b296d84318f0da5dbe4bba69ff2bf0fe489 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 24 Jul 2024 18:42:44 -0400 Subject: [PATCH 3/5] fix(swingset): update initialization API from PR feedback makeKernelKeeper() now takes the expected version, and throws if it doesn't match. This replaces the version check in kernel.js start(). --- .../src/controller/initializeKernel.js | 10 +-- .../src/controller/initializeSwingset.js | 5 +- .../src/controller/upgradeSwingset.js | 49 +++++++----- packages/SwingSet/src/kernel/kernel.js | 11 ++- .../SwingSet/src/kernel/state/kernelKeeper.js | 72 ++++++++++------- packages/SwingSet/test/clist.test.js | 26 ++++-- packages/SwingSet/test/controller.test.js | 6 +- .../SwingSet/test/snapshots/state.test.js.md | 4 +- .../test/snapshots/state.test.js.snap | Bin 276 -> 279 bytes packages/SwingSet/test/state.test.js | 74 ++++++++++-------- .../SwingSet/test/transcript-light.test.js | 2 +- .../SwingSet/test/upgrade-swingset.test.js | 12 ++- 12 files changed, 164 insertions(+), 107 deletions(-) diff --git a/packages/SwingSet/src/controller/initializeKernel.js b/packages/SwingSet/src/controller/initializeKernel.js index e0f70a44690..aa84c4b0216 100644 --- a/packages/SwingSet/src/controller/initializeKernel.js +++ b/packages/SwingSet/src/controller/initializeKernel.js @@ -43,15 +43,9 @@ export async function initializeKernel(config, kernelStorage, options = {}) { const logStartup = verbose ? console.debug : () => 0; insistStorageAPI(kernelStorage.kvStore); - const CURRENT_VERSION = 1; - kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); - - const kernelSlog = null; - const kernelKeeper = makeKernelKeeper(kernelStorage, kernelSlog); + const kernelKeeper = makeKernelKeeper(kernelStorage, 'uninitialized'); const optionRecorder = makeVatOptionRecorder(kernelKeeper, bundleHandler); - const wasInitialized = kernelKeeper.getInitialized(); - assert(!wasInitialized); const { defaultManagerType, defaultReapInterval = DEFAULT_DELIVERIES_PER_BOYD, @@ -93,7 +87,7 @@ export async function initializeKernel(config, kernelStorage, options = {}) { // generate the genesis vats await null; - if (config.vats) { + if (config.vats && Object.keys(config.vats).length) { for (const name of Object.keys(config.vats)) { const { bundleID, diff --git a/packages/SwingSet/src/controller/initializeSwingset.js b/packages/SwingSet/src/controller/initializeSwingset.js index 2ea889af3e4..be85b28e066 100644 --- a/packages/SwingSet/src/controller/initializeSwingset.js +++ b/packages/SwingSet/src/controller/initializeSwingset.js @@ -256,7 +256,10 @@ export async function loadSwingsetConfigFile(configPath) { } export function swingsetIsInitialized(kernelStorage) { - return !!kernelStorage.kvStore.get('initialized'); + return !!( + kernelStorage.kvStore.get('version') || + kernelStorage.kvStore.get('initialized') + ); } /** diff --git a/packages/SwingSet/src/controller/upgradeSwingset.js b/packages/SwingSet/src/controller/upgradeSwingset.js index 3c194b5ffa5..5de665342b7 100644 --- a/packages/SwingSet/src/controller/upgradeSwingset.js +++ b/packages/SwingSet/src/controller/upgradeSwingset.js @@ -30,23 +30,29 @@ const upgradeVatV0toV1 = (kvStore, defaultReapDirtThreshold, vatID) => { assert(kvStore.has(oldReapCountdownKey), oldReapCountdownKey); assert(!kvStore.has(reapDirtKey), reapDirtKey); - // initialize or upgrade state - const reapDirt = {}; // all missing keys are treated as zero - const threshold = {}; - const reapIntervalString = kvStore.get(oldReapIntervalKey); - assert(reapIntervalString !== undefined); const reapCountdownString = kvStore.get(oldReapCountdownKey); + assert(reapIntervalString !== undefined); assert(reapCountdownString !== undefined); + const intervalIsNever = reapIntervalString === 'never'; const countdownIsNever = reapCountdownString === 'never'; + // these were supposed to be the same assert( - (intervalIsNever && countdownIsNever) || - (!intervalIsNever && !countdownIsNever), + intervalIsNever === countdownIsNever, `reapInterval=${reapIntervalString}, reapCountdown=${reapCountdownString}`, ); - if (!intervalIsNever && !countdownIsNever) { + // initialize or upgrade state + const reapDirt = {}; // all missing keys are treated as zero + const threshold = {}; + + if (intervalIsNever) { + // old vats that were never reaped (eg comms) used + // reapInterval='never', so respect that and set the other + // threshold values to never as well + threshold.never = true; + } else { // deduce delivery count from old countdown values const reapInterval = Number.parseInt(reapIntervalString, 10); const reapCountdown = Number.parseInt(reapCountdownString, 10); @@ -57,17 +63,11 @@ const upgradeVatV0toV1 = (kvStore, defaultReapDirtThreshold, vatID) => { } } - // old vats that were never reaped (eg comms) used - // reapInterval='never', so respect that and set the other - // threshold values to never as well - if (intervalIsNever) { - threshold.never = true; - } kvStore.delete(oldReapIntervalKey); kvStore.delete(oldReapCountdownKey); kvStore.set(reapDirtKey, JSON.stringify(reapDirt)); - // remove .reapInterval from options, replace with .reapDirtThreshold + // Update options to use the new schema. const options = JSON.parse(kvStore.get(vatOptionsKey)); delete options.reapInterval; options.reapDirtThreshold = threshold; @@ -91,8 +91,8 @@ const upgradeVatV0toV1 = (kvStore, defaultReapDirtThreshold, vatID) => { * called during the upgrade, and it is responsible for doing a * `hostStorage.commit()` afterwards. * - * @param { SwingStoreKernelStorage } kernelStorage - * @returns { boolean } true if any changes were made + * @param {SwingStoreKernelStorage} kernelStorage + * @returns {boolean} true if any changes were made */ export const upgradeSwingset = kernelStorage => { const { kvStore } = kernelStorage; @@ -120,7 +120,10 @@ export const upgradeSwingset = kernelStorage => { // steps applied here must match. // schema v0: - // The kernel overall has `kernel.defaultReapInterval`. + // + // The kernel overall has `kernel.defaultReapInterval` and + // `kernel.initialized = 'true'`. + // // Each vat has a `vNN.reapInterval` and `vNN.reapCountdown`. // vNN.options has a `.reapInterval` property (however it was not // updated by processChangeVatOptions, so do not rely upon its @@ -128,17 +131,25 @@ export const upgradeSwingset = kernelStorage => { if (version < 1) { // schema v1: - // The kernel overall has `kernel.defaultReapDirtThreshold`. + // + // The kernel overall has `kernel.defaultReapDirtThreshold` and + // `kernel.version = '1'` (instead of .initialized). + // // Each vat has a `vNN.reapDirt`, and vNN.options has a // `.reapDirtThreshold` property // So: + // * replace `kernel.initialized` with `kernel.version` // * replace `kernel.defaultReapInterval` with // `kernel.defaultReapDirtThreshold` // * replace vat's `vNN.reapInterval`/`vNN.reapCountdown` with // `vNN.reapDirt` and a `vNN.reapDirtThreshold` in `vNN.options` // * then do per-vat upgrades with upgradeVatV0toV1 + assert(kvStore.has('initialized')); + kvStore.delete('initialized'); + // 'version' will be set at the end + // upgrade from old kernel.defaultReapInterval const oldDefaultReapIntervalKey = 'kernel.defaultReapInterval'; diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js index 036f968a7bb..032847ada77 100644 --- a/packages/SwingSet/src/kernel/kernel.js +++ b/packages/SwingSet/src/kernel/kernel.js @@ -10,7 +10,9 @@ import { foreverPolicy } from '../lib/runPolicies.js'; import { makeVatManagerFactory } from './vat-loader/manager-factory.js'; import { makeVatWarehouse } from './vat-warehouse.js'; import makeDeviceManager from './deviceManager.js'; -import makeKernelKeeper from './state/kernelKeeper.js'; +import makeKernelKeeper, { + CURRENT_SCHEMA_VERSION, +} from './state/kernelKeeper.js'; import { kdebug, kdebugEnable, @@ -112,7 +114,11 @@ export default function buildKernel( ? makeSlogger(slogCallbacks, writeSlogObject) : makeDummySlogger(slogCallbacks, makeConsole('disabled slogger')); - const kernelKeeper = makeKernelKeeper(kernelStorage, kernelSlog); + const kernelKeeper = makeKernelKeeper( + kernelStorage, + CURRENT_SCHEMA_VERSION, + kernelSlog, + ); /** @type {ReturnType} */ let vatWarehouse; @@ -1662,7 +1668,6 @@ export default function buildKernel( throw Error('kernel.start already called'); } started = true; - kernelKeeper.getInitialized() || Fail`kernel not initialized`; kernelKeeper.loadStats(); diff --git a/packages/SwingSet/src/kernel/state/kernelKeeper.js b/packages/SwingSet/src/kernel/state/kernelKeeper.js index e5a1db20c41..d1e149c44be 100644 --- a/packages/SwingSet/src/kernel/state/kernelKeeper.js +++ b/packages/SwingSet/src/kernel/state/kernelKeeper.js @@ -49,6 +49,9 @@ const enableKernelGC = true; export { DEFAULT_REAP_DIRT_THRESHOLD_KEY }; +// most recent DB schema version +export const CURRENT_SCHEMA_VERSION = 1; + // Kernel state lives in a key-value store supporting key retrieval by // lexicographic range. All keys and values are strings. // We simulate a tree by concatenating path-name components with ".". When we @@ -60,7 +63,13 @@ export { DEFAULT_REAP_DIRT_THRESHOLD_KEY }; // allowed to vary between instances in a consensus machine. Everything else // is required to be deterministic. // -// The current ("v1") schema is: +// +// The schema is indicated by the value of the "version" key, which +// was added for version 1 (i.e., version 0 had no such key), and is +// only modified by a call to upgradeSwingset(). See below for +// deltas/upgrades from one version to the next. +// +// The current ("v1") schema keys/values are: // // version = '1' // vat.names = JSON([names..]) @@ -152,12 +161,16 @@ export { DEFAULT_REAP_DIRT_THRESHOLD_KEY }; // Prefix reserved for host written data: // host. -// Kernel state schemas. The 'version' key records the state of the -// database, and is only modified by a call to upgradeSwingset(). +// Kernel state schema changes: // // v0: the original -// v1: replace `kernel.defaultReapInterval` with `kernel.defaultReapDirtThreshold` -// replace vat's `vNN.reapInterval`/`vNN.reapCountdown` with `vNN.reapDirt` +// * no `version` +// * uses `initialized = 'true'` +// v1: +// * add `version = '1'` +// * remove `initialized` +// * replace `kernel.defaultReapInterval` with `kernel.defaultReapDirtThreshold` +// * replace vat's `vNN.reapInterval`/`vNN.reapCountdown` with `vNN.reapDirt` // and a `vNN.reapDirtThreshold` in `vNN.options` export function commaSplit(s) { @@ -221,17 +234,35 @@ export const DEFAULT_DELIVERIES_PER_BOYD = 1; export const DEFAULT_GC_KREFS_PER_BOYD = 20; -const EXPECTED_VERSION = 1; - /** * @param {SwingStoreKernelStorage} kernelStorage - * @param {KernelSlog|null} kernelSlog + * @param {number | 'uninitialized'} expectedVersion + * @param {KernelSlog} [kernelSlog] */ -export default function makeKernelKeeper(kernelStorage, kernelSlog) { +export default function makeKernelKeeper( + kernelStorage, + expectedVersion, + kernelSlog, +) { const { kvStore, transcriptStore, snapStore, bundleStore } = kernelStorage; insistStorageAPI(kvStore); + const versionString = kvStore.get('version'); + const version = Number(versionString || '0'); + if (expectedVersion === 'uninitialized') { + if (kvStore.has('initialized')) { + throw Error(`kernel DB already initialized (v0)`); + } + if (versionString) { + throw Error(`kernel DB already initialized (v${versionString})`); + } + } else if (expectedVersion !== version) { + throw Error( + `kernel DB is too old: has version v${version}, but expected v${expectedVersion}`, + ); + } + /** * @param {string} key * @returns {string} @@ -242,16 +273,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { return kvStore.get(key); } - if ( - !kvStore.has('version') || - Number(getRequired('version')) !== EXPECTED_VERSION - ) { - const have = kvStore.get('version') || 'undefined'; - throw Error( - `kernelStorage is too old (have ${have}, need ${EXPECTED_VERSION}), please upgradeSwingset()`, - ); - } - const { incStat, decStat, @@ -303,12 +324,10 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { deviceKeepers: new Map(), // deviceID -> deviceKeeper }); - function getInitialized() { - return !!kvStore.get('initialized'); - } - function setInitialized() { - kvStore.set('initialized', 'true'); + assert(!kvStore.has('initialized')); + assert(!kvStore.has('version')); + kvStore.set('version', `${CURRENT_SCHEMA_VERSION}`); } function getCrankNumber() { @@ -430,14 +449,14 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { /** * - * @returns { ReapDirtThreshold } + * @returns {ReapDirtThreshold} */ function getDefaultReapDirtThreshold() { return JSON.parse(getRequired(DEFAULT_REAP_DIRT_THRESHOLD_KEY)); } /** - * @param { ReapDirtThreshold } threshold + * @param {ReapDirtThreshold} threshold */ function setDefaultReapDirtThreshold(threshold) { assert.typeof(threshold, 'object'); @@ -1602,7 +1621,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { } return harden({ - getInitialized, setInitialized, createStartingKernelState, getDefaultManagerType, diff --git a/packages/SwingSet/test/clist.test.js b/packages/SwingSet/test/clist.test.js index df2be62545b..1210fb3afc4 100644 --- a/packages/SwingSet/test/clist.test.js +++ b/packages/SwingSet/test/clist.test.js @@ -4,17 +4,22 @@ import { test } from '../tools/prepare-test-env-ava.js'; // eslint-disable-next-line import/order import { initSwingStore } from '@agoric/swing-store'; import { makeDummySlogger } from '../src/kernel/slogger.js'; -import makeKernelKeeper from '../src/kernel/state/kernelKeeper.js'; - -const CURRENT_VERSION = 1; +import makeKernelKeeper, { + CURRENT_SCHEMA_VERSION, +} from '../src/kernel/state/kernelKeeper.js'; test(`clist reachability`, async t => { const slog = makeDummySlogger({}); const kernelStorage = initSwingStore(null).kernelStorage; - kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); - const kk = makeKernelKeeper(kernelStorage, slog); + const k0 = makeKernelKeeper(kernelStorage, 'uninitialized'); + k0.createStartingKernelState({ defaultManagerType: 'local' }); + k0.setInitialized(); + k0.saveStats(); + + const kk = makeKernelKeeper(kernelStorage, CURRENT_SCHEMA_VERSION, slog); + kk.loadStats(); + const s = kk.kvStore; - kk.createStartingKernelState({ defaultManagerType: 'local' }); const vatID = kk.allocateUnusedVatID(); const source = { bundleID: 'foo' }; const options = { workerOptions: {}, reapDirtThreshold: {} }; @@ -100,8 +105,13 @@ test(`clist reachability`, async t => { test('getImporters', async t => { const slog = makeDummySlogger({}); const kernelStorage = initSwingStore(null).kernelStorage; - kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); - const kk = makeKernelKeeper(kernelStorage, slog); + const k0 = makeKernelKeeper(kernelStorage, 'uninitialized'); + k0.createStartingKernelState({ defaultManagerType: 'local' }); + k0.setInitialized(); + k0.saveStats(); + + const kk = makeKernelKeeper(kernelStorage, CURRENT_SCHEMA_VERSION, slog); + kk.loadStats(); kk.createStartingKernelState({ defaultManagerType: 'local' }); const vatID1 = kk.allocateUnusedVatID(); diff --git a/packages/SwingSet/test/controller.test.js b/packages/SwingSet/test/controller.test.js index 6994081b4ea..47464616ddb 100644 --- a/packages/SwingSet/test/controller.test.js +++ b/packages/SwingSet/test/controller.test.js @@ -11,7 +11,9 @@ import { initializeSwingset, makeSwingsetController, } from '../src/index.js'; -import makeKernelKeeper from '../src/kernel/state/kernelKeeper.js'; +import makeKernelKeeper, { + CURRENT_SCHEMA_VERSION, +} from '../src/kernel/state/kernelKeeper.js'; import { checkKT } from './util.js'; const emptyVP = kser({}); @@ -494,7 +496,7 @@ test('comms vat does not BOYD', async t => { const kernelStorage = initSwingStore().kernelStorage; const controller = await buildVatController(config, [], { kernelStorage }); t.teardown(controller.shutdown); - const k = makeKernelKeeper(kernelStorage, null); + const k = makeKernelKeeper(kernelStorage, CURRENT_SCHEMA_VERSION); const commsVatID = k.getVatIDForName('comms'); t.deepEqual( JSON.parse(k.kvStore.get(`${commsVatID}.options`)).reapDirtThreshold, diff --git a/packages/SwingSet/test/snapshots/state.test.js.md b/packages/SwingSet/test/snapshots/state.test.js.md index 5a7cae18e2a..cdaf5c03bdc 100644 --- a/packages/SwingSet/test/snapshots/state.test.js.md +++ b/packages/SwingSet/test/snapshots/state.test.js.md @@ -8,8 +8,8 @@ Generated by [AVA](https://avajs.dev). > initial state - '2cc47b69a725bb4a2bfca1e2ba2b8625e3a62261acac60e37be95ebc09b1e02e' + '99068f7796b6004cfad57c5079365643b0306f525e529ef83f3b80b424517bff' > expected activityhash - 'c7edd8883ba896276247c1de6391d1cdac3fcc6bfbd1599098dbd367e454b41f' + '62b78f65e37e063b3ac5982eff14077e178970080a8338a5bf5476570f168a4d' diff --git a/packages/SwingSet/test/snapshots/state.test.js.snap b/packages/SwingSet/test/snapshots/state.test.js.snap index 0a7a22b266fec1f45e928a1857a0de7f00ee0290..55ffcea9b2da4965aa9010d7dba4d08f10f0a6ca 100644 GIT binary patch literal 279 zcmV+y0qFigRzVIBLdcu>|1v0D|odPLI`>P zaMD(pPalg200000000ARkTFgJF%U&JLWr6hY)6GOyY_hOjuTMRj_nz(6=yfX289xC zLQYB|DkOgSoA>_z)>yZ+Z=YxBrB^vUq*q?|3%6x_9LrjI;X1a?T}EE>$T?rCZEdBk zd~D0s+t)c~?)>t6Gu#c=>~4qq-BSqQyb=W$9e_>B87U!9FbW!M{ zA==sEL{U{LN2!ND3jdSb9LcLgF(lwjG@LN_SyU$rL^6>N1mJHAM@d~S8&faurGH#c_yI@Wlo22S000#?cz^%^ literal 276 zcmV+v0qg!jRzV7L*p4dF`mug=oPe74e_3nA@kZF7P{K{f zN!f@BiK#}@d^7W%u&i`!pI7O*7r8vrE7aoxZ5a<^uDNF@W9!gmgvui@Z_!#?ZWYdL zS$q4sGDC-#=eyy4xM{Z^9(GTf$a-`Uyq*y)XiNl3G;S_9J&OUSwNpSqZW7~B;w(s0 zL?x3JDS%Q=e=7V>A}`D#1cMNw)81+8NwGK+6)R#OlZYH9WHrY)#lXnMd9kxatI{v& aO)h7lF55=x`91fK+X+7>P-~hX0RRAo7 { ]); }); -const CURRENT_VERSION = 1; - function buildKeeperStorageInMemory() { const { kernelStorage, debug } = initSwingStore(null); - kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); return { ...debug, // serialize, dump ...kernelStorage, @@ -158,7 +157,7 @@ function buildKeeperStorageInMemory() { function duplicateKeeper(serialize) { const serialized = serialize(); const { kernelStorage } = initSwingStore(null, { serialized }); - const kernelKeeper = makeKernelKeeper(kernelStorage, null); + const kernelKeeper = makeKernelKeeper(kernelStorage, CURRENT_SCHEMA_VERSION); kernelKeeper.loadStats(); return kernelKeeper; } @@ -178,16 +177,14 @@ test('kernelStorage param guards', async t => { test('kernel state', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); - t.truthy(!k.getInitialized()); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); k.setInitialized(); - k.emitCrankHashes(); + checkState(t, store.dump, [ ['version', '1'], ['crankNumber', '0'], - ['initialized', 'true'], ['gcActions', '[]'], ['runQueue', '[1,1]'], ['acceptanceQueue', '[1,1]'], @@ -214,8 +211,9 @@ test('kernel state', async t => { test('kernelKeeper vat names', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); const v1 = k.allocateVatIDForNameIfNeeded('vatname5'); const v2 = k.allocateVatIDForNameIfNeeded('Frank'); @@ -268,8 +266,9 @@ test('kernelKeeper vat names', async t => { test('kernelKeeper device names', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); const d7 = k.allocateDeviceIDForNameIfNeeded('devicename5'); const d8 = k.allocateDeviceIDForNameIfNeeded('Frank'); @@ -322,8 +321,9 @@ test('kernelKeeper device names', async t => { test('kernelKeeper runQueue', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); t.is(k.getRunQueueLength(), 0); t.is(k.getNextRunQueueMsg(), undefined); @@ -360,8 +360,9 @@ test('kernelKeeper runQueue', async t => { test('kernelKeeper promises', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); const p1 = k.addKernelPromiseForVat('v4'); t.deepEqual(k.getKernelPromise(p1), { @@ -497,8 +498,9 @@ test('kernelKeeper promises', async t => { test('kernelKeeper promise resolveToData', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); const p1 = k.addKernelPromiseForVat('v4'); const o1 = k.addKernelObject('v1'); @@ -513,8 +515,9 @@ test('kernelKeeper promise resolveToData', async t => { test('kernelKeeper promise reject', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); const p1 = k.addKernelPromiseForVat('v4'); const o1 = k.addKernelObject('v1'); @@ -529,8 +532,9 @@ test('kernelKeeper promise reject', async t => { test('vatKeeper', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); const v1 = k.allocateVatIDForNameIfNeeded('name1'); const source = { bundleID: 'foo' }; const options = { workerOptions: {}, reapDirtThreshold: {} }; @@ -568,8 +572,9 @@ test('vatKeeper', async t => { test('vatKeeper.getOptions', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); const v1 = k.allocateVatIDForNameIfNeeded('name1'); const bundleID = 'b1-00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'; @@ -585,15 +590,17 @@ test('vatKeeper.getOptions', async t => { test('XS vatKeeper defaultManagerType', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'xs-worker' }); + k.setInitialized(); t.is(k.getDefaultManagerType(), 'xs-worker'); }); test('meters', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); const m1 = k.allocateMeter(100n, 10n); const m2 = k.allocateMeter(200n, 150n); t.not(m1, m2); @@ -677,8 +684,9 @@ const makeTestCrankHasher = (algorithm = 'sha256') => { test('crankhash - initial state and additions', t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); k.emitCrankHashes(); // the initial state additions happen to hash to this: const initialActivityHash = store.kvStore.get('activityhash'); @@ -716,8 +724,9 @@ Then commit the changes in .../snapshots/ path. test('crankhash - skip keys', t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); k.emitCrankHashes(); k.kvStore.set('one', '1'); @@ -743,8 +752,9 @@ test('crankhash - duplicate set', t => { // hash as we add/delete, not just the accumulated additions/deletions set const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); k.emitCrankHashes(); k.kvStore.set('one', '1'); @@ -774,8 +784,9 @@ test('crankhash - set and delete', t => { // setting and deleting a key is different than never setting it const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); + const k = makeKernelKeeper(store, 'uninitialized'); k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); k.emitCrankHashes(); const h1 = makeTestCrankHasher('sha256'); @@ -948,10 +959,9 @@ test('stats - can load and save existing stats', t => { test('vatKeeper dirt counters', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); - k.createStartingKernelState({ - defaultManagerType: 'local', - }); + const k = makeKernelKeeper(store, 'uninitialized'); + k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); k.saveStats(); // the defaults are designed for testing @@ -1029,10 +1039,9 @@ test('vatKeeper dirt counters', async t => { test('dirt upgrade', async t => { const store = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(store, null); - k.createStartingKernelState({ - defaultManagerType: 'local', - }); + const k = makeKernelKeeper(store, 'uninitialized'); + k.createStartingKernelState({ defaultManagerType: 'local' }); + k.setInitialized(); k.saveStats(); const v1 = k.allocateVatIDForNameIfNeeded('name1'); const source = { bundleID: 'foo' }; @@ -1088,6 +1097,7 @@ test('dirt upgrade', async t => { k.kvStore.set(`${v3}.reapCountdown`, 'never'); k.kvStore.delete(`version`); + k.kvStore.set('initialized', 'true'); // kernelKeeper refuses to work with an old state t.throws(() => duplicateKeeper(store.serialize)); @@ -1098,7 +1108,7 @@ test('dirt upgrade', async t => { const serialized = store.serialize(); const { kernelStorage } = initSwingStore(null, { serialized }); upgradeSwingset(kernelStorage); - k2 = makeKernelKeeper(kernelStorage, null); // works this time + k2 = makeKernelKeeper(kernelStorage, CURRENT_SCHEMA_VERSION); // works this time k2.loadStats(); } diff --git a/packages/SwingSet/test/transcript-light.test.js b/packages/SwingSet/test/transcript-light.test.js index bc934ec478a..fa56e6a0a58 100644 --- a/packages/SwingSet/test/transcript-light.test.js +++ b/packages/SwingSet/test/transcript-light.test.js @@ -17,7 +17,7 @@ test('transcript-light load', async t => { t.teardown(c.shutdown); const serialized0 = debug.serialize(); const kvstate0 = debug.dump().kvEntries; - t.is(kvstate0.initialized, 'true'); + t.is(kvstate0.version, '1'); t.is(kvstate0.runQueue, '[1,1]'); t.not(kvstate0.acceptanceQueue, '[]'); diff --git a/packages/SwingSet/test/upgrade-swingset.test.js b/packages/SwingSet/test/upgrade-swingset.test.js index 1a468543bf5..11558bc7d47 100644 --- a/packages/SwingSet/test/upgrade-swingset.test.js +++ b/packages/SwingSet/test/upgrade-swingset.test.js @@ -25,15 +25,17 @@ test('kernel refuses to run with out-of-date DB', async t => { await commit(); // now doctor the initial state to make it look like the - // kernelkeeper v0 schema, just deleting the version key + // kernelkeeper v0 schema, just deleting the version key and adding + // 'initialized' t.is(kvStore.get('version'), '1'); kvStore.delete(`version`); + kvStore.set('initialized', 'true'); await commit(); // Now build a controller around this modified state, which should fail. await t.throwsAsync(() => makeSwingsetController(kernelStorage), { - message: /kernelStorage is too old/, + message: /kernel DB is too old/, }); }); @@ -73,6 +75,7 @@ test('upgrade kernel state', async t => { t.is(kvStore.get('version'), '1'); kvStore.delete('version'); // i.e. revert to v0 + kvStore.set('initialized', 'true'); kvStore.delete(`kernel.defaultReapDirtThreshold`); kvStore.set(`kernel.defaultReapInterval`, '300'); @@ -103,7 +106,7 @@ test('upgrade kernel state', async t => { // confirm that this state is too old for the kernel to use await t.throwsAsync(() => makeSwingsetController(kernelStorage), { - message: /kernelStorage is too old/, + message: /kernel DB is too old/, }); // upgrade it @@ -161,6 +164,7 @@ test('upgrade non-reaping kernel state', async t => { t.is(kvStore.get('version'), '1'); kvStore.delete('version'); // i.e. revert to v0 + kvStore.set('initialized', 'true'); kvStore.delete(`kernel.defaultReapDirtThreshold`); kvStore.set(`kernel.defaultReapInterval`, 'never'); @@ -184,7 +188,7 @@ test('upgrade non-reaping kernel state', async t => { // confirm that this state is too old for the kernel to use await t.throwsAsync(() => makeSwingsetController(kernelStorage), { - message: /kernelStorage is too old/, + message: /kernel DB is too old/, }); // upgrade it From c7696069d0bebaf039a2f3e1a45ebdd8dc5198a2 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Tue, 2 Jul 2024 16:04:56 -0700 Subject: [PATCH 4/5] fix(cosmic-swingset): call upgradeSwingset at startup Any changes will be committed as part of the first block after reboot. This should be in-consensus, because all nodes are supposed to change kernel versions at the same time, and only during a chain-halting upgrade. --- packages/cosmic-swingset/src/launch-chain.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index 25ae404c880..8c1353e4d16 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -21,6 +21,7 @@ import { makeSwingsetController, loadBasedir, loadSwingsetConfigFile, + upgradeSwingset, } from '@agoric/swingset-vat'; import { waitUntilQuiescent } from '@agoric/internal/src/lib-nodejs/waitUntilQuiescent.js'; import { openSwingStore } from '@agoric/swing-store'; @@ -214,6 +215,7 @@ export async function buildSwingset( } const coreProposals = await ensureSwingsetInitialized(); + upgradeSwingset(kernelStorage); const controller = await makeSwingsetController( kernelStorage, deviceEndowments, From 6547c8318d83ca58704a4c911608706c25795c68 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 10 Jul 2024 08:58:31 -0700 Subject: [PATCH 5/5] fix(cosmic-swingset): add exportCallback interlock The swingstore export-data callback gives us export-data records, which must be written into IAVL by sending them over to the golang side with swingStoreExportCallback . However, that callback isn't ready right away, so if e.g. openSwingStore() were to invoke it, we might lose those records. Likewise saveOutsideState() gathers the chainSends just before calling commit, so if the callback were invoked during commit(), those records would be left for a subsequent block, which would break consensus if the node crashed before the next commit. This commit adds an `allowExportCallback` flag, to catch these two cases. It is enabled at the start of AG_COSMOS_INIT and BEGIN_BLOCK, and then disabled just before we flush the chain sends in saveOutsideState() (called during COMMIT_BLOCK). Note that swingstore is within its rights to call exportCallback during openSwingStore() or commit(), it just happens to not do so right now. If that changes under maintenance, this guard should turn a corruption bug into a crash bug refs #9655 --- packages/cosmic-swingset/src/launch-chain.js | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index 8c1353e4d16..ad1620a855d 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -333,6 +333,25 @@ export async function launch({ }) { console.info('Launching SwingSet kernel'); + // The swingstore export-data callback gives us export-data records, + // which must be written into IAVL by sending them over to the + // golang side with swingStoreExportCallback . However, that + // callback isn't ready right away, so if e.g. openSwingStore() were + // to invoke it, we might lose those records. Likewise + // saveOutsideState() gathers the chainSends just before calling + // commit, so if the callback were invoked during commit(), those + // records would be left for a subsequent block, which would break + // consensus if the node crashed before the next commit. So this + // `allowExportCallback` flag serves to catch these two cases. + // + // Note that swingstore is within its rights to call exportCallback + // during openSwingStore() or commit(), it just happens to not do so + // right now. If that changes under maintenance, this guard should + // turn a corruption bug into a crash bug. See + // https://github.com/Agoric/agoric-sdk/issues/9655 for details + + let allowExportCallback = false; + // The swingStore's exportCallback is synchronous, however we allow the // callback provided to launch-chain to be asynchronous. The callbacks are // invoked sequentially like if they were awaited, and the block manager @@ -343,6 +362,7 @@ export async function launch({ const swingStoreExportSyncCallback = swingStoreExportCallback && (updates => { + assert(allowExportCallback, 'export-data callback called at bad time'); pendingSwingStoreExport = swingStoreExportCallbackWithQueue(updates); }); @@ -470,6 +490,7 @@ export async function launch({ } async function saveOutsideState(blockHeight) { + allowExportCallback = false; const chainSends = await clearChainSends(); kvStore.set(getHostKey('height'), `${blockHeight}`); kvStore.set(getHostKey('chainSends'), JSON.stringify(chainSends)); @@ -889,6 +910,7 @@ export async function launch({ // ); switch (action.type) { case ActionType.AG_COSMOS_INIT: { + allowExportCallback = true; // cleared by saveOutsideState in COMMIT_BLOCK const { blockHeight, isBootstrap, upgradeDetails } = action; if (!blockNeedsExecution(blockHeight)) { @@ -975,6 +997,7 @@ export async function launch({ } case ActionType.BEGIN_BLOCK: { + allowExportCallback = true; // cleared by saveOutsideState in COMMIT_BLOCK const { blockHeight, blockTime, params } = action; blockParams = parseParams(params); verboseBlocks &&