diff --git a/packages/kernel-test/src/persistence.test.ts b/packages/kernel-test/src/persistence.test.ts index a498639b6..e004bbee2 100644 --- a/packages/kernel-test/src/persistence.test.ts +++ b/packages/kernel-test/src/persistence.test.ts @@ -47,9 +47,9 @@ describe('persistent storage', { timeout: 20_000 }, () => { }; it('maintains state across kernel restarts', async () => { - const database = await makeSQLKernelDatabase({ dbFilename: databasePath }); + const database1 = await makeSQLKernelDatabase({ dbFilename: databasePath }); const kernel1 = await makeKernel( - database, + database1, false, logger.logger.subLogger({ tags: ['test'] }), ); @@ -60,14 +60,16 @@ describe('persistent storage', { timeout: 20_000 }, () => { expect(incrementResult1).toBe('Counter incremented to: 2'); await waitUntilQuiescent(); await kernel1.stop(); + const database2 = await makeSQLKernelDatabase({ dbFilename: databasePath }); const kernel2 = await makeKernel( - database, + database2, false, logger.logger.subLogger({ tags: ['test'] }), ); await new Promise((resolve) => setTimeout(resolve, 1000)); const resumeResult = await runResume(kernel2, v1Root); expect(resumeResult).toBe('Counter incremented to: 3'); + await kernel2.stop(); }); it('handles multiple vats with persistent state', async () => { @@ -88,9 +90,9 @@ describe('persistent storage', { timeout: 20_000 }, () => { }, }, }; - const database = await makeSQLKernelDatabase({ dbFilename: databasePath }); + const database1 = await makeSQLKernelDatabase({ dbFilename: databasePath }); const kernel1 = await makeKernel( - database, + database1, false, logger.logger.subLogger({ tags: ['test'] }), ); @@ -101,14 +103,16 @@ describe('persistent storage', { timeout: 20_000 }, () => { expect(workResult1).toBe('Work completed: Worker1(1), Worker2(1)'); await waitUntilQuiescent(); await kernel1.stop(); + const database2 = await makeSQLKernelDatabase({ dbFilename: databasePath }); const kernel2 = await makeKernel( - database, + database2, false, logger.logger.subLogger({ tags: ['test'] }), ); await new Promise((resolve) => setTimeout(resolve, 1000)); const workResult2 = await runResume(kernel2, v1Root); expect(workResult2).toBe('Work completed: Worker1(2), Worker2(2)'); + await kernel2.stop(); }); it('respects resetStorage flag when set to true', async () => { @@ -148,10 +152,10 @@ describe('persistent storage', { timeout: 20_000 }, () => { }); it('handles messages in queue after kernel restart', async () => { - const database = await makeSQLKernelDatabase({ dbFilename: databasePath }); - const kernelStore = makeKernelStore(database); + const database1 = await makeSQLKernelDatabase({ dbFilename: databasePath }); + const kernelStore1 = makeKernelStore(database1); const kernel1 = await makeKernel( - database, + database1, false, logger.logger.subLogger({ tags: ['test'] }), ); @@ -164,31 +168,34 @@ describe('persistent storage', { timeout: 20_000 }, () => { const result1 = await kernel1.queueMessage(v1Root, 'resume', []); expect(kunser(result1)).toBe('Counter incremented to: 2'); // Enqueue a send message into the database - kernelStore.kv.set('queue.run.head', '4'); - kernelStore.kv.set('nextPromiseId', '4'); - kernelStore.kv.set(`${v1Root}.refCount`, '3,3'); - kernelStore.kv.set('queue.kp3.head', '1'); - kernelStore.kv.set('queue.kp3.tail', '1'); - kernelStore.kv.set('kp3.state', 'unresolved'); - kernelStore.kv.set('kp3.subscribers', '[]'); - kernelStore.kv.set('kp3.refCount', '2'); - kernelStore.kv.set( + kernelStore1.kv.set('queue.run.head', '4'); + kernelStore1.kv.set('nextPromiseId', '4'); + kernelStore1.kv.set(`${v1Root}.refCount`, '3,3'); + kernelStore1.kv.set('queue.kp3.head', '1'); + kernelStore1.kv.set('queue.kp3.tail', '1'); + kernelStore1.kv.set('kp3.state', 'unresolved'); + kernelStore1.kv.set('kp3.subscribers', '[]'); + kernelStore1.kv.set('kp3.refCount', '2'); + kernelStore1.kv.set( 'queue.run.3', `{"type":"send","target":"${v1Root}","message":{"methargs":{"body":"#[\\"resume\\",[]]","slots":[]},"result":"kp3"}}`, ); await kernel1.stop(); - // verify that the message is in the database - expect(kernelStore.kv.get('queue.run.3')).toBeDefined(); + // Open a fresh connection to verify the message is in the database + const database2 = await makeSQLKernelDatabase({ dbFilename: databasePath }); + const kernelStore2 = makeKernelStore(database2); + expect(kernelStore2.kv.get('queue.run.3')).toBeDefined(); // restart the kernel const kernel2 = await makeKernel( - database, + database2, false, logger.logger.subLogger({ tags: ['test'] }), ); // verify that the run queue is empty - expect(kernelStore.kv.get('queue.run.3')).toBeUndefined(); + expect(kernelStore2.kv.get('queue.run.3')).toBeUndefined(); // verify that the message is processed and the counter is incremented const result2 = await kernel2.queueMessage(v1Root, 'resume', []); expect(kunser(result2)).toBe('Counter incremented to: 4'); + await kernel2.stop(); }); }); diff --git a/packages/kernel-test/src/remote-comms.test.ts b/packages/kernel-test/src/remote-comms.test.ts index aae2e1250..66c1c5f2a 100644 --- a/packages/kernel-test/src/remote-comms.test.ts +++ b/packages/kernel-test/src/remote-comms.test.ts @@ -13,7 +13,10 @@ import type { RemoteCommsOptions, } from '@metamask/ocap-kernel'; import { NodejsPlatformServices } from '@ocap/nodejs'; -import { describe, it, expect, beforeEach } from 'vitest'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { makeTestLogger, @@ -278,6 +281,17 @@ describe('Remote Communications (Integration Tests)', () => { ); }); + afterEach(async () => { + await Promise.all([ + kernel1.stop().catch(() => { + // already stopped inside the test + }), + kernel2.stop().catch(() => { + // already stopped inside the test + }), + ]); + }); + it('should initialize remote communications without errors', async () => { const status1 = await kernel1.getStatus(); const status2 = await kernel2.getStatus(); @@ -365,87 +379,120 @@ describe('Remote Communications (Integration Tests)', () => { }); it('remote relationships should survive kernel restart', async () => { - // Launch client vat on kernel1 - const clientConfig = makeMaasClientConfig('client1', true); - let clientKernel = kernel1; - await runTestVats(clientKernel, clientConfig); - const clientRootRef = kernelStore1.getRootObject('v1') as KRef; - - // Launch server vat on kernel2 - const serverConfig = makeMaasServerConfig('server2', true); - let serverKernel = kernel2; - const serverResult = await runTestVats(serverKernel, serverConfig); - - // The server's ocap URL is its bootstrap result - const serverURL = serverResult as string; - - expect(typeof serverURL).toBe('string'); - expect(serverURL).toMatch(/^ocap:/u); - - // Configure the client with the server's URL - const setupResult = await clientKernel.queueMessage( - clientRootRef, - 'setMaas', - [serverURL], - ); - let response = kunser(setupResult); - expect(response).toBeDefined(); - expect(response).toContain('MaaS service URL set'); - - // Tell the client to talk to the server - let expectedCount = 1; - const stepResult = await clientKernel.queueMessage( - clientRootRef, - 'step', - [], - ); - response = kunser(stepResult); - expect(response).toBeDefined(); - expect(response).toContain(`next step: ${expectedCount} `); - - // Kill the server and restart it - await serverKernel.stop(); - serverKernel = await makeTestKernel( - 'kernel2b', - kernelDatabase2, - directNetwork, - false, - 'kernel2-peer', - '02', - ); - - // Tell the client to talk to the server a second time - expectedCount += 1; - const stepResult2 = await clientKernel.queueMessage( - clientRootRef, - 'step', - [], - ); - response = kunser(stepResult2); - expect(response).toBeDefined(); - expect(response).toContain(`next step: ${expectedCount} `); - - // Kill the client and restart it - await clientKernel.stop(); - clientKernel = await makeTestKernel( - 'kernel1b', - kernelDatabase1, - directNetwork, - false, - 'kernel1-peer', - '01', - ); - - // Tell the client to talk to the server a third time - expectedCount += 1; - const stepResult3 = await clientKernel.queueMessage( - clientRootRef, - 'step', - [], - ); - response = kunser(stepResult3); - expect(response).toBeDefined(); - expect(response).toContain(`next step: ${expectedCount} `); + // This test needs file-based databases for persistence across stop/restart. + // Stop the beforeEach kernels and replace them with file-backed ones. + const tempDir = await mkdtemp(join(tmpdir(), 'kernel-test-rc-')); + const dbFile1 = join(tempDir, 'k1.db'); + const dbFile2 = join(tempDir, 'k2.db'); + try { + await Promise.all([kernel1.stop(), kernel2.stop()]); + + const initDb1 = await makeSQLKernelDatabase({ dbFilename: dbFile1 }); + const localKernelStore1 = makeKernelStore(initDb1); + let clientKernel = await makeTestKernel( + 'kernel1', + initDb1, + directNetwork, + true, + 'kernel1-peer', + '01', + ); + + const initDb2 = await makeSQLKernelDatabase({ dbFilename: dbFile2 }); + let serverKernel = await makeTestKernel( + 'kernel2', + initDb2, + directNetwork, + true, + 'kernel2-peer', + '02', + ); + + // Launch client vat on kernel1 + const clientConfig = makeMaasClientConfig('client1', true); + await runTestVats(clientKernel, clientConfig); + const clientRootRef = localKernelStore1.getRootObject('v1') as KRef; + + // Launch server vat on kernel2 + const serverConfig = makeMaasServerConfig('server2', true); + const serverResult = await runTestVats(serverKernel, serverConfig); + + // The server's ocap URL is its bootstrap result + const serverURL = serverResult as string; + + expect(typeof serverURL).toBe('string'); + expect(serverURL).toMatch(/^ocap:/u); + + // Configure the client with the server's URL + const setupResult = await clientKernel.queueMessage( + clientRootRef, + 'setMaas', + [serverURL], + ); + let response = kunser(setupResult); + expect(response).toBeDefined(); + expect(response).toContain('MaaS service URL set'); + + // Tell the client to talk to the server + let expectedCount = 1; + const stepResult = await clientKernel.queueMessage( + clientRootRef, + 'step', + [], + ); + response = kunser(stepResult); + expect(response).toBeDefined(); + expect(response).toContain(`next step: ${expectedCount} `); + + // Kill the server and restart it with a fresh db connection + await serverKernel.stop(); + serverKernel = await makeTestKernel( + 'kernel2b', + await makeSQLKernelDatabase({ dbFilename: dbFile2 }), + directNetwork, + false, + 'kernel2-peer', + '02', + ); + + // Tell the client to talk to the server a second time + expectedCount += 1; + const stepResult2 = await clientKernel.queueMessage( + clientRootRef, + 'step', + [], + ); + response = kunser(stepResult2); + expect(response).toBeDefined(); + expect(response).toContain(`next step: ${expectedCount} `); + + // Kill the client and restart it with a fresh db connection + await clientKernel.stop(); + clientKernel = await makeTestKernel( + 'kernel1b', + await makeSQLKernelDatabase({ dbFilename: dbFile1 }), + directNetwork, + false, + 'kernel1-peer', + '01', + ); + + // Tell the client to talk to the server a third time + expectedCount += 1; + const stepResult3 = await clientKernel.queueMessage( + clientRootRef, + 'step', + [], + ); + response = kunser(stepResult3); + expect(response).toBeDefined(); + expect(response).toContain(`next step: ${expectedCount} `); + + // Stop the local kernels before the temp dir is cleaned up + await Promise.all([clientKernel.stop(), serverKernel.stop()]); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } }); }); diff --git a/packages/nodejs/test/e2e/bip39-identity-recovery.test.ts b/packages/nodejs/test/e2e/bip39-identity-recovery.test.ts index beb32d07f..a63c4ca4c 100644 --- a/packages/nodejs/test/e2e/bip39-identity-recovery.test.ts +++ b/packages/nodejs/test/e2e/bip39-identity-recovery.test.ts @@ -1,6 +1,9 @@ import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; import { Kernel } from '@metamask/ocap-kernel'; import type { KernelStatus } from '@metamask/ocap-kernel'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, it, expect } from 'vitest'; import { makeTestKernel } from '../helpers/kernel.ts'; @@ -62,7 +65,6 @@ describe('BIP39 Identity Recovery', () => { if (kernel1) { await kernel1.stop(); } - kernelDatabase1.close(); } // Create fresh database and kernel with same mnemonic @@ -88,7 +90,6 @@ describe('BIP39 Identity Recovery', () => { if (kernel2) { await kernel2.stop(); } - kernelDatabase2.close(); } }, TEST_TIMEOUT, @@ -117,7 +118,6 @@ describe('BIP39 Identity Recovery', () => { if (kernel1) { await kernel1.stop(); } - kernelDatabase1.close(); } // Create kernel with different mnemonic @@ -142,7 +142,6 @@ describe('BIP39 Identity Recovery', () => { if (kernel2) { await kernel2.stop(); } - kernelDatabase2.close(); } }, TEST_TIMEOUT, @@ -151,25 +150,29 @@ describe('BIP39 Identity Recovery', () => { it( 'throws error when mnemonic provided but identity already exists in storage', async () => { - const kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); + const tempDir3 = await mkdtemp(join(tmpdir(), 'ocap-bip39-')); + const dbFilename3 = join(tempDir3, 'kernel.db'); let kernel: Kernel | undefined; try { // First kernel without mnemonic - generates random identity - kernel = await makeTestKernel(kernelDatabase); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: dbFilename3 }), + ); await kernel.initRemoteComms({ relays: DUMMY_RELAYS }); const status1 = await kernel.getStatus(); expect(getRemoteCommsPeerId(status1.remoteComms)).toBeDefined(); - // Stop kernel but don't close database + // Stop kernel (also closes database) await kernel.stop(); kernel = undefined; // Create kernel with mnemonic but using existing storage - should throw - kernel = await makeTestKernel(kernelDatabase, { resetStorage: false }); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: dbFilename3 }), + { resetStorage: false }, + ); await expect( kernel.initRemoteComms({ relays: DUMMY_RELAYS, @@ -182,7 +185,7 @@ describe('BIP39 Identity Recovery', () => { if (kernel) { await kernel.stop(); } - kernelDatabase.close(); + await rm(tempDir3, { recursive: true, force: true }); } }, TEST_TIMEOUT, @@ -209,7 +212,6 @@ describe('BIP39 Identity Recovery', () => { if (kernel) { await kernel.stop(); } - kernelDatabase.close(); } }, TEST_TIMEOUT, @@ -218,28 +220,30 @@ describe('BIP39 Identity Recovery', () => { it( 'allows recovery with resetStorage and mnemonic when identity exists', async () => { - const kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); + const tempDir5 = await mkdtemp(join(tmpdir(), 'ocap-bip39-')); + const dbFilename5 = join(tempDir5, 'kernel.db'); let kernel: Kernel | undefined; try { // First kernel without mnemonic - generates random identity - kernel = await makeTestKernel(kernelDatabase); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: dbFilename5 }), + ); await kernel.initRemoteComms({ relays: DUMMY_RELAYS }); const status1 = await kernel.getStatus(); const originalPeerId = getRemoteCommsPeerId(status1.remoteComms); expect(originalPeerId).toBeDefined(); - // Stop kernel but don't close database + // Stop kernel (also closes database) await kernel.stop(); kernel = undefined; // Create kernel with resetStorage AND mnemonic - should work - kernel = await makeTestKernel(kernelDatabase, { - mnemonic: TEST_MNEMONIC, - }); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: dbFilename5 }), + { mnemonic: TEST_MNEMONIC }, + ); await kernel.initRemoteComms({ relays: DUMMY_RELAYS }); const status2 = await kernel.getStatus(); @@ -253,9 +257,10 @@ describe('BIP39 Identity Recovery', () => { await kernel.stop(); kernel = undefined; - kernel = await makeTestKernel(kernelDatabase, { - mnemonic: TEST_MNEMONIC, - }); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: dbFilename5 }), + { mnemonic: TEST_MNEMONIC }, + ); await kernel.initRemoteComms({ relays: DUMMY_RELAYS }); const status3 = await kernel.getStatus(); @@ -264,7 +269,7 @@ describe('BIP39 Identity Recovery', () => { if (kernel) { await kernel.stop(); } - kernelDatabase.close(); + await rm(tempDir5, { recursive: true, force: true }); } }, TEST_TIMEOUT, diff --git a/packages/nodejs/test/e2e/remote-comms.test.ts b/packages/nodejs/test/e2e/remote-comms.test.ts index 15c082618..c44075080 100644 --- a/packages/nodejs/test/e2e/remote-comms.test.ts +++ b/packages/nodejs/test/e2e/remote-comms.test.ts @@ -5,6 +5,9 @@ import { Kernel, kunser, makeKernelStore } from '@metamask/ocap-kernel'; import type { KRef } from '@metamask/ocap-kernel'; import { startRelay } from '@ocap/cli/relay'; import { delay } from '@ocap/repo-tools/test-utils'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { makeTestKernel, runTestVats } from '../helpers/kernel.ts'; @@ -55,8 +58,9 @@ describe.sequential('Remote Communications E2E', () => { let relay: Libp2p; let kernel1: Kernel; let kernel2: Kernel; - let kernelDatabase1: Awaited>; - let kernelDatabase2: Awaited>; + let dbFilename1: string; + let dbFilename2: string; + let tempDir: string; let kernelStore1: ReturnType; let kernelStore2: ReturnType; @@ -66,14 +70,19 @@ describe.sequential('Remote Communications E2E', () => { // Wait for relay to be fully initialized await delay(1000); + // Create temp directory for database files + tempDir = await mkdtemp(join(tmpdir(), 'ocap-e2e-')); + dbFilename1 = join(tempDir, 'kernel1.db'); + dbFilename2 = join(tempDir, 'kernel2.db'); + // Create two independent kernels with separate storage - kernelDatabase1 = await makeSQLKernelDatabase({ - dbFilename: ':memory:', + const kernelDatabase1 = await makeSQLKernelDatabase({ + dbFilename: dbFilename1, }); kernelStore1 = makeKernelStore(kernelDatabase1); - kernelDatabase2 = await makeSQLKernelDatabase({ - dbFilename: ':memory:', + const kernelDatabase2 = await makeSQLKernelDatabase({ + dbFilename: dbFilename2, }); kernelStore2 = makeKernelStore(kernelDatabase2); @@ -100,11 +109,8 @@ describe.sequential('Remote Communications E2E', () => { 'kernel2.stop', ), ]); - if (kernelDatabase1) { - kernelDatabase1.close(); - } - if (kernelDatabase2) { - kernelDatabase2.close(); + if (tempDir) { + await rm(tempDir, { recursive: true, force: true }); } await delay(200); }); @@ -232,9 +238,10 @@ describe.sequential('Remote Communications E2E', () => { // Kill the server and restart it await serverKernel.stop(); - serverKernel = await makeTestKernel(kernelDatabase2, { - resetStorage: false, - }); + serverKernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: dbFilename2 }), + { resetStorage: false }, + ); await serverKernel.initRemoteComms({ relays: testRelays }); // Tell the client to talk to the server a second time @@ -250,9 +257,10 @@ describe.sequential('Remote Communications E2E', () => { // Kill the client and restart it await clientKernel.stop(); - clientKernel = await makeTestKernel(kernelDatabase1, { - resetStorage: false, - }); + clientKernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: dbFilename1 }), + { resetStorage: false }, + ); await clientKernel.initRemoteComms({ relays: testRelays }); // Tell the client to talk to the server a third time @@ -265,6 +273,12 @@ describe.sequential('Remote Communications E2E', () => { response = kunser(stepResult3); expect(response).toBeDefined(); expect(response).toContain(`next step: ${expectedCount} `); + + // Update describe-scope refs so afterEach stops the restarted kernels + // eslint-disable-next-line require-atomic-updates + kernel1 = clientKernel; + // eslint-disable-next-line require-atomic-updates + kernel2 = serverKernel; }, NETWORK_TIMEOUT * 2, ); @@ -302,15 +316,14 @@ describe.sequential('Remote Communications E2E', () => { // Restart kernel2 - the queued message should trigger reconnection const bobConfig = makeRemoteVatConfig('Bob'); + const restartResult = await restartKernelAndReloadVat( + dbFilename2, + false, + testRelays, + bobConfig, + ); // eslint-disable-next-line require-atomic-updates - kernel2 = ( - await restartKernelAndReloadVat( - kernelDatabase2, - false, - testRelays, - bobConfig, - ) - ).kernel; + kernel2 = restartResult.kernel; // Wait for the recovery message to complete const recoveryResult = await recoveryPromise; @@ -405,15 +418,14 @@ describe.sequential('Remote Communications E2E', () => { const reconnectStartTime = Date.now(); const bobConfig = makeRemoteVatConfig('Bob'); + const restartResult = await restartKernelAndReloadVat( + dbFilename2, + false, + testRelays, + bobConfig, + ); // eslint-disable-next-line require-atomic-updates - kernel2 = ( - await restartKernelAndReloadVat( - kernelDatabase2, - false, - testRelays, - bobConfig, - ) - ).kernel; + kernel2 = restartResult.kernel; // The queued message should now be delivered const reconnectResult = await messagePromise; @@ -469,15 +481,14 @@ describe.sequential('Remote Communications E2E', () => { ]); queuePromises.push(promise); } + const restartResult = await restartKernelAndReloadVat( + dbFilename2, + false, + testRelays, + bobConfig, + ); // eslint-disable-next-line require-atomic-updates - kernel2 = ( - await restartKernelAndReloadVat( - kernelDatabase2, - false, - testRelays, - bobConfig, - ) - ).kernel; + kernel2 = restartResult.kernel; // Messages should be queued and delivered after reconnection // Note: Some may fail if the vat wasn't restored properly, but queueing should work @@ -558,15 +569,14 @@ describe.sequential('Remote Communications E2E', () => { } const bobConfig = makeRemoteVatConfig('Bob'); + const restartResult = await restartKernelAndReloadVat( + dbFilename2, + false, + testRelays, + bobConfig, + ); // eslint-disable-next-line require-atomic-updates - kernel2 = ( - await restartKernelAndReloadVat( - kernelDatabase2, - false, - testRelays, - bobConfig, - ) - ).kernel; + kernel2 = restartResult.kernel; // Check results - messages beyond queue capacity should be rejected const results = await Promise.allSettled(messagePromises); @@ -605,8 +615,9 @@ describe.sequential('Remote Communications E2E', () => { 'handles multiple simultaneous reconnections to different peers', async () => { // Create a third kernel for testing multiple peers + const dbFilename3 = join(tempDir, 'kernel3.db'); const kernelDatabase3 = await makeSQLKernelDatabase({ - dbFilename: ':memory:', + dbFilename: dbFilename3, }); let kernel3: Kernel | undefined; @@ -653,19 +664,18 @@ describe.sequential('Remote Communications E2E', () => { const bobConfigRestart = makeRemoteVatConfig('Bob'); const charlieConfigRestart = makeRemoteVatConfig('Charlie'); + const restartResult2 = await restartKernelAndReloadVat( + dbFilename2, + false, + testRelays, + bobConfigRestart, + ); // eslint-disable-next-line require-atomic-updates - kernel2 = ( - await restartKernelAndReloadVat( - kernelDatabase2, - false, - testRelays, - bobConfigRestart, - ) - ).kernel; + kernel2 = restartResult2.kernel; kernel3 = ( await restartKernelAndReloadVat( - kernelDatabase3, + dbFilename3, false, testRelays, charlieConfigRestart, @@ -704,9 +714,6 @@ describe.sequential('Remote Communications E2E', () => { if (kernel3) { await kernel3.stop(); } - if (kernelDatabase3) { - kernelDatabase3.close(); - } } }, NETWORK_TIMEOUT * 3, @@ -744,15 +751,14 @@ describe.sequential('Remote Communications E2E', () => { ); const bobConfig = makeRemoteVatConfig('Bob'); + const restartResult = await restartKernelAndReloadVat( + dbFilename2, + false, + testRelays, + bobConfig, + ); // eslint-disable-next-line require-atomic-updates - kernel2 = ( - await restartKernelAndReloadVat( - kernelDatabase2, - false, - testRelays, - bobConfig, - ) - ).kernel; + kernel2 = restartResult.kernel; // The message should not have been delivered because we didn't reconnect const result = await messageAfterClose; @@ -790,15 +796,14 @@ describe.sequential('Remote Communications E2E', () => { await kernel1.reconnectPeer(peerId2); const bobConfig = makeRemoteVatConfig('Bob'); + const restartResult = await restartKernelAndReloadVat( + dbFilename2, + false, + testRelays, + bobConfig, + ); // eslint-disable-next-line require-atomic-updates - kernel2 = ( - await restartKernelAndReloadVat( - kernelDatabase2, - false, - testRelays, - bobConfig, - ) - ).kernel; + kernel2 = restartResult.kernel; const messageAfterReconnect = await sendRemoteMessage( kernel1, @@ -893,19 +898,18 @@ describe.sequential('Remote Communications E2E', () => { // Establish connection and exchange handshakes await sendRemoteMessage(kernel1, aliceRef, bobURL, 'hello', ['Alice']); - // Stop kernel2 + // Stop kernel2 (also closes the database) await kernel2.stop(); - // Simulate state loss by closing kernel2's database and creating fresh in-memory db - kernelDatabase2.close(); - // eslint-disable-next-line require-atomic-updates - kernelDatabase2 = await makeSQLKernelDatabase({ - dbFilename: ':memory:', + // Simulate state loss by creating a fresh database (new incarnation ID, no previous state) + const freshDb2 = await makeSQLKernelDatabase({ + dbFilename: join(tempDir, 'kernel2-fresh.db'), }); // Create a completely new kernel (new incarnation ID, no previous state) + const freshKernel2 = await makeTestKernel(freshDb2); // eslint-disable-next-line require-atomic-updates - kernel2 = await makeTestKernel(kernelDatabase2); + kernel2 = freshKernel2; await kernel2.initRemoteComms({ relays: testRelays }); // Launch Bob again (fresh vat, no previous state) @@ -1007,15 +1011,14 @@ describe.sequential('Remote Communications E2E', () => { // Restart kernel2 quickly (before max retries, since default is infinite) // The promise should remain unresolved and resolve normally after reconnection const bobConfig = makeRemoteVatConfig('Bob'); + const restartResult = await restartKernelAndReloadVat( + dbFilename2, + false, + testRelays, + bobConfig, + ); // eslint-disable-next-line require-atomic-updates - kernel2 = ( - await restartKernelAndReloadVat( - kernelDatabase2, - false, - testRelays, - bobConfig, - ) - ).kernel; + kernel2 = restartResult.kernel; // Wait for reconnection await delay(2000); diff --git a/packages/nodejs/test/e2e/system-subcluster.test.ts b/packages/nodejs/test/e2e/system-subcluster.test.ts index b30e8c6ca..f54ee627c 100644 --- a/packages/nodejs/test/e2e/system-subcluster.test.ts +++ b/packages/nodejs/test/e2e/system-subcluster.test.ts @@ -1,4 +1,3 @@ -import type { KernelDatabase } from '@metamask/kernel-store'; import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; import { Kernel, kunser } from '@metamask/ocap-kernel'; import type { @@ -6,6 +5,9 @@ import type { ClusterConfig, } from '@metamask/ocap-kernel'; import { delay } from '@ocap/repo-tools/test-utils'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, it, expect, afterEach } from 'vitest'; import { makeTestKernel } from '../helpers/kernel.ts'; @@ -15,7 +17,6 @@ const SAMPLE_VAT_BUNDLE_URL = 'http://localhost:3000/sample-vat.bundle'; describe('System Subcluster', { timeout: 30_000 }, () => { let kernel: Kernel | undefined; - let kernelDatabase: KernelDatabase | undefined; const makeSystemSubclusterConfig = ( name: string, @@ -40,32 +41,24 @@ describe('System Subcluster', { timeout: 30_000 }, () => { kernel = undefined; await stopResult; } - if (kernelDatabase) { - kernelDatabase.close(); - kernelDatabase = undefined; - } }); describe('initialization', () => { it('launches system subcluster at kernel initialization', async () => { - kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); - kernel = await makeTestKernel(kernelDatabase, { - systemSubclusters: [makeSystemSubclusterConfig('test-system')], - }); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: ':memory:' }), + { systemSubclusters: [makeSystemSubclusterConfig('test-system')] }, + ); // System subcluster's bootstrap vat should be running expect(kernel.getVatIds().length).toBeGreaterThan(0); }); it('provides bootstrap root via getSystemSubclusterRoot', async () => { - kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); - kernel = await makeTestKernel(kernelDatabase, { - systemSubclusters: [makeSystemSubclusterConfig('test-system')], - }); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: ':memory:' }), + { systemSubclusters: [makeSystemSubclusterConfig('test-system')] }, + ); const root = kernel.getSystemSubclusterRoot('test-system'); expect(root).toBeDefined(); @@ -76,12 +69,10 @@ describe('System Subcluster', { timeout: 30_000 }, () => { describe('kernel services', () => { it('receives kernelFacet service in bootstrap', async () => { - kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); - kernel = await makeTestKernel(kernelDatabase, { - systemSubclusters: [makeSystemSubclusterConfig('test-system')], - }); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: ':memory:' }), + { systemSubclusters: [makeSystemSubclusterConfig('test-system')] }, + ); const root = kernel.getSystemSubclusterRoot('test-system'); expect(root).toBeDefined(); @@ -93,12 +84,10 @@ describe('System Subcluster', { timeout: 30_000 }, () => { }); it('queries kernel status via kernelFacet', async () => { - kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); - kernel = await makeTestKernel(kernelDatabase, { - systemSubclusters: [makeSystemSubclusterConfig('test-system')], - }); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: ':memory:' }), + { systemSubclusters: [makeSystemSubclusterConfig('test-system')] }, + ); const root = kernel.getSystemSubclusterRoot('test-system'); expect(root).toBeDefined(); @@ -118,12 +107,10 @@ describe('System Subcluster', { timeout: 30_000 }, () => { describe('subcluster management', () => { it('launches subcluster via kernelFacet', async () => { - kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); - kernel = await makeTestKernel(kernelDatabase, { - systemSubclusters: [makeSystemSubclusterConfig('test-system')], - }); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: ':memory:' }), + { systemSubclusters: [makeSystemSubclusterConfig('test-system')] }, + ); const root = kernel.getSystemSubclusterRoot('test-system'); expect(root).toBeDefined(); @@ -161,12 +148,10 @@ describe('System Subcluster', { timeout: 30_000 }, () => { }); it('terminates subcluster via kernelFacet', async () => { - kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); - kernel = await makeTestKernel(kernelDatabase, { - systemSubclusters: [makeSystemSubclusterConfig('test-system')], - }); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: ':memory:' }), + { systemSubclusters: [makeSystemSubclusterConfig('test-system')] }, + ); const root = kernel.getSystemSubclusterRoot('test-system'); expect(root).toBeDefined(); @@ -214,123 +199,145 @@ describe('System Subcluster', { timeout: 30_000 }, () => { describe('system subcluster persistence', () => { it('restores existing system subcluster on kernel restart', async () => { - kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); - kernel = await makeTestKernel(kernelDatabase, { - systemSubclusters: [makeSystemSubclusterConfig('test-system')], - }); - - // Get initial subcluster info - const initialSubclusters = kernel.getSubclusters(); - expect(initialSubclusters).toHaveLength(1); - const initialSubclusterId = initialSubclusters[0]!.id; - const initialRoot = kernel.getSystemSubclusterRoot('test-system'); - expect(initialRoot).toBeDefined(); - - // Stop kernel but keep database - await kernel.stop(); - - // Restart kernel with same system subcluster config (resetStorage = false) - // eslint-disable-next-line require-atomic-updates - kernel = await makeTestKernel(kernelDatabase, { - resetStorage: false, - systemSubclusters: [makeSystemSubclusterConfig('test-system')], - }); - - // System subcluster should be restored (not relaunched) - const newSubclusters = kernel.getSubclusters(); - expect(newSubclusters).toHaveLength(1); - const newSubclusterId = newSubclusters[0]!.id; - - // Subcluster ID should be the SAME (restored from persistence) - expect(newSubclusterId).toBe(initialSubclusterId); - - // Bootstrap root should be restored - const newRoot = kernel.getSystemSubclusterRoot('test-system'); - expect(newRoot).toBeDefined(); - expect(newRoot).toBe(initialRoot); - - const result = await kernel.queueMessage(newRoot, 'hasKernelFacet', []); - await delay(); - expect(kunser(result)).toBe(true); + const tempDir = await mkdtemp(join(tmpdir(), 'ocap-ss-')); + const dbFilename = join(tempDir, 'kernel.db'); + try { + let initialSubclusterId: string; + let initialRoot: string | undefined; + + const firstKernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename }), + { systemSubclusters: [makeSystemSubclusterConfig('test-system')] }, + ); + try { + const initialSubclusters = firstKernel.getSubclusters(); + expect(initialSubclusters).toHaveLength(1); + initialSubclusterId = initialSubclusters[0]!.id; + initialRoot = firstKernel.getSystemSubclusterRoot('test-system'); + expect(initialRoot).toBeDefined(); + } finally { + await firstKernel.stop(); + } + + const secondKernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename }), + { + resetStorage: false, + systemSubclusters: [makeSystemSubclusterConfig('test-system')], + }, + ); + try { + const newSubclusters = secondKernel.getSubclusters(); + expect(newSubclusters).toHaveLength(1); + expect(newSubclusters[0]!.id).toBe(initialSubclusterId); + + const newRoot = secondKernel.getSystemSubclusterRoot('test-system'); + expect(newRoot).toBeDefined(); + expect(newRoot).toBe(initialRoot); + + const result = await secondKernel.queueMessage( + newRoot, + 'hasKernelFacet', + [], + ); + await delay(); + expect(kunser(result)).toBe(true); + } finally { + await secondKernel.stop(); + } + } finally { + await rm(tempDir, { recursive: true, force: true }); + } }); it('persists baggage data across kernel restarts', async () => { - kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); - kernel = await makeTestKernel(kernelDatabase, { - systemSubclusters: [makeSystemSubclusterConfig('test-system')], - }); - - const root = kernel.getSystemSubclusterRoot('test-system'); - expect(root).toBeDefined(); - - // Store data in baggage during first incarnation - const testKey = 'persistent-data'; - const testValue = 'hello from first incarnation'; - await kernel.queueMessage(root, 'storeToBaggage', [testKey, testValue]); - await delay(); - - // Verify data was stored - const storedResult = await kernel.queueMessage(root, 'getFromBaggage', [ - testKey, - ]); - await delay(); - expect(kunser(storedResult)).toBe(testValue); - - // Stop kernel but keep database - await kernel.stop(); - - // Restart kernel with same system subcluster config (resetStorage = false) - // eslint-disable-next-line require-atomic-updates - kernel = await makeTestKernel(kernelDatabase, { - resetStorage: false, - systemSubclusters: [makeSystemSubclusterConfig('test-system')], - }); - - // Get restored root after restart (should be the same as before) - const newRoot = kernel.getSystemSubclusterRoot('test-system'); - expect(newRoot).toBeDefined(); - expect(newRoot).toBe(root); - - // Verify baggage data persisted across restart - const persistedResult = await kernel.queueMessage( - newRoot, - 'getFromBaggage', - [testKey], - ); - await delay(); - expect(kunser(persistedResult)).toBe(testValue); - - // Verify key exists check works - const hasKeyResult = await kernel.queueMessage(newRoot, 'hasBaggageKey', [ - testKey, - ]); - await delay(); - expect(kunser(hasKeyResult)).toBe(true); - - // Verify non-existent key returns false - const noKeyResult = await kernel.queueMessage(newRoot, 'hasBaggageKey', [ - 'non-existent-key', - ]); - await delay(); - expect(kunser(noKeyResult)).toBe(false); + const tempDir = await mkdtemp(join(tmpdir(), 'ocap-ss-')); + const dbFilename = join(tempDir, 'kernel.db'); + try { + let root: string | undefined; + const testKey = 'persistent-data'; + const testValue = 'hello from first incarnation'; + + const firstKernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename }), + { systemSubclusters: [makeSystemSubclusterConfig('test-system')] }, + ); + try { + root = firstKernel.getSystemSubclusterRoot('test-system'); + expect(root).toBeDefined(); + + await firstKernel.queueMessage(root, 'storeToBaggage', [ + testKey, + testValue, + ]); + await delay(); + + const storedResult = await firstKernel.queueMessage( + root, + 'getFromBaggage', + [testKey], + ); + await delay(); + expect(kunser(storedResult)).toBe(testValue); + } finally { + await firstKernel.stop(); + } + + const secondKernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename }), + { + resetStorage: false, + systemSubclusters: [makeSystemSubclusterConfig('test-system')], + }, + ); + try { + const newRoot = secondKernel.getSystemSubclusterRoot('test-system'); + expect(newRoot).toBeDefined(); + expect(newRoot).toBe(root); + + const persistedResult = await secondKernel.queueMessage( + newRoot, + 'getFromBaggage', + [testKey], + ); + await delay(); + expect(kunser(persistedResult)).toBe(testValue); + + const hasKeyResult = await secondKernel.queueMessage( + newRoot, + 'hasBaggageKey', + [testKey], + ); + await delay(); + expect(kunser(hasKeyResult)).toBe(true); + + const noKeyResult = await secondKernel.queueMessage( + newRoot, + 'hasBaggageKey', + ['non-existent-key'], + ); + await delay(); + expect(kunser(noKeyResult)).toBe(false); + } finally { + await secondKernel.stop(); + } + } finally { + await rm(tempDir, { recursive: true, force: true }); + } }); }); describe('multiple system subclusters', () => { it('launches multiple system subclusters at kernel initialization', async () => { - kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); - kernel = await makeTestKernel(kernelDatabase, { - systemSubclusters: [ - makeSystemSubclusterConfig('system-1'), - makeSystemSubclusterConfig('system-2'), - ], - }); + kernel = await makeTestKernel( + await makeSQLKernelDatabase({ dbFilename: ':memory:' }), + { + systemSubclusters: [ + makeSystemSubclusterConfig('system-1'), + makeSystemSubclusterConfig('system-2'), + ], + }, + ); // Both system subclusters should have bootstrap roots const root1 = kernel.getSystemSubclusterRoot('system-1'); diff --git a/packages/nodejs/test/helpers/remote-comms.ts b/packages/nodejs/test/helpers/remote-comms.ts index b8c3be0bf..09feb2f1b 100644 --- a/packages/nodejs/test/helpers/remote-comms.ts +++ b/packages/nodejs/test/helpers/remote-comms.ts @@ -1,4 +1,4 @@ -import type { KernelDatabase } from '@metamask/kernel-store'; +import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs'; import { stringify } from '@metamask/kernel-utils'; import { Kernel, kunser, makeKernelStore } from '@metamask/ocap-kernel'; import type { @@ -132,18 +132,19 @@ export async function sendRemoteMessage( } /** - * Restart a kernel with the same database. + * Restart a kernel by opening a fresh database connection to the same file. * - * @param kernelDatabase - The kernel database to use. + * @param dbFilename - The database filename to open a fresh connection to. * @param resetStorage - Whether to reset storage. * @param relays - Array of relay addresses. * @returns The restarted kernel. */ export async function restartKernel( - kernelDatabase: KernelDatabase, + dbFilename: string, resetStorage: boolean, relays: string[], ): Promise { + const kernelDatabase = await makeSQLKernelDatabase({ dbFilename }); const kernel = await makeTestKernel(kernelDatabase, { resetStorage }); await kernel.initRemoteComms({ relays }); return kernel; @@ -152,19 +153,19 @@ export async function restartKernel( /** * Restart a kernel and relaunch its vat. * - * @param kernelDatabase - The kernel database to use. + * @param dbFilename - The database filename to open a fresh connection to. * @param resetStorage - Whether to reset storage. * @param relays - Array of relay addresses. * @param config - Cluster configuration for the vat. * @returns Object with the restarted kernel and its ocap URL. */ export async function restartKernelAndReloadVat( - kernelDatabase: KernelDatabase, + dbFilename: string, resetStorage: boolean, relays: string[], config: ClusterConfig, ): Promise<{ kernel: Kernel; url: string }> { - const kernel = await restartKernel(kernelDatabase, resetStorage, relays); + const kernel = await restartKernel(dbFilename, resetStorage, relays); const url = await launchVatAndGetURL(kernel, config); return { kernel, url }; } diff --git a/packages/ocap-kernel/src/Kernel.ts b/packages/ocap-kernel/src/Kernel.ts index 4a5296aa4..850ce69bf 100644 --- a/packages/ocap-kernel/src/Kernel.ts +++ b/packages/ocap-kernel/src/Kernel.ts @@ -84,6 +84,9 @@ export class Kernel { /** The kernel's router */ readonly #kernelRouter: KernelRouter; + /** Database holding the kernel's persistent state */ + readonly #kernelDatabase: KernelDatabase; + /** Manages IO channel lifecycle (optional, requires factory injection) */ readonly #ioManager: IOManager | undefined; @@ -112,6 +115,7 @@ export class Kernel { } = {}, ) { this.#platformServices = platformServices; + this.#kernelDatabase = kernelDatabase; this.#logger = options.logger ?? new Logger('ocap-kernel'); this.#kernelStore = makeKernelStore(kernelDatabase, this.#logger); if (!this.#kernelStore.kv.get('initialized')) { @@ -737,6 +741,7 @@ export class Kernel { await this.#platformServices.stopRemoteComms(); this.#remoteManager.cleanup(); await this.#platformServices.terminateAll(); + this.#kernelDatabase.close(); } /** diff --git a/packages/ocap-kernel/test/storage.ts b/packages/ocap-kernel/test/storage.ts index 41f9abd2e..67814b5c5 100644 --- a/packages/ocap-kernel/test/storage.ts +++ b/packages/ocap-kernel/test/storage.ts @@ -127,6 +127,9 @@ export function makeMapKernelDatabase(): KernelDatabase { deleteVatStore: (vatID: string) => { vatStores.delete(vatID); }, + close: () => { + // noop + }, createSavepoint: () => { // noop },