diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e0ff051a08..5d50ed8952 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -535,6 +535,7 @@ jobs: "server-endpoints/search-test.ts", "server-endpoints/info-test.ts", "server-endpoints/search-prerendered-test.ts", + "server-config-test.ts", "server-endpoints/stripe-session-test.ts", "server-endpoints/stripe-webhook-test.ts", "server-endpoints/user-and-catalog-test.ts", diff --git a/packages/matrix/docker/synapse/index.ts b/packages/matrix/docker/synapse/index.ts index 979cc56357..eebd8c9043 100644 --- a/packages/matrix/docker/synapse/index.ts +++ b/packages/matrix/docker/synapse/index.ts @@ -1,6 +1,7 @@ import * as path from 'path'; import * as os from 'os'; import * as crypto from 'crypto'; +import * as net from 'net'; import * as fse from 'fs-extra'; import { request } from '@playwright/test'; import { @@ -40,14 +41,58 @@ export interface SynapseInstance extends SynapseConfig { } const synapses = new Map(); +const dynamicHostPortStartAttempts = 5; + +function findAvailablePort(preferred?: number): Promise { + return new Promise((resolve, reject) => { + let server = net.createServer(); + + server.on('error', (error: NodeJS.ErrnoException) => { + server.close(); + if (preferred != null && error.code === 'EADDRINUSE') { + findAvailablePort(undefined).then(resolve, reject); + return; + } + reject(error); + }); + + server.listen(preferred ?? 0, '127.0.0.1', () => { + let address = server.address(); + if (!address || typeof address === 'string') { + server.close(() => + reject(new Error('Could not determine available port')), + ); + return; + } + let { port } = address; + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(port); + }); + }); + }); +} function randB64Bytes(numBytes: number): string { return crypto.randomBytes(numBytes).toString('base64').replace(/=*$/, ''); } +function isPortBindError(error: unknown): boolean { + let message = error instanceof Error ? error.message : String(error); + return /address already in use|port is already allocated/i.test(message); +} + export async function cfgDirFromTemplate( template: string, dataDir?: string, + options?: { + publicBaseUrl?: string; + host?: string; + port?: number; + }, ): Promise { const templateDir = path.join(__dirname, template); @@ -69,7 +114,9 @@ export async function cfgDirFromTemplate( const macaroonSecret = randB64Bytes(16); const formSecret = randB64Bytes(16); - const baseUrl = `http://${SYNAPSE_IP_ADDRESS}:${SYNAPSE_PORT}`; + const host = options?.host ?? SYNAPSE_IP_ADDRESS; + const port = options?.port ?? SYNAPSE_PORT; + const baseUrl = options?.publicBaseUrl ?? `http://${host}:${port}`; // now copy homeserver.yaml, applying substitutions console.log(`Gen ${path.join(templateDir, 'homeserver.yaml')}`); @@ -95,8 +142,8 @@ export async function cfgDirFromTemplate( ); return { - port: SYNAPSE_PORT, - host: SYNAPSE_IP_ADDRESS, + port, + host, baseUrl, configDir, registrationSecret, @@ -136,51 +183,80 @@ export async function synapseStart( } await Promise.allSettled(stopPromises); } - const synCfg = await cfgDirFromTemplate( - opts?.template ?? 'test', - opts?.dataDir, - ); - let containerName = - opts?.containerName || - (isEnvironmentMode() - ? getSynapseContainerName() - : path.basename(synCfg.configDir)); - console.log( - `Starting synapse with config dir ${synCfg.configDir} in container ${containerName}...`, + let useDynamicHostPort = Boolean( + isEnvironmentMode() || opts?.dynamicHostPort, ); await dockerCreateNetwork({ networkName: 'boxel' }); - let dockerParams: string[] = [ - '--rm', - '-v', - `${synCfg.configDir}:/data`, - '-v', - `${path.join(__dirname, 'templates')}:/custom/templates/`, - ]; - if (isEnvironmentMode() || opts?.dynamicHostPort) { - // Dynamic host port, with fixed container IP only when not running in branch mode - if (!isEnvironmentMode()) { - dockerParams.push(`--ip=${synCfg.host}`); - } - dockerParams.push('-p', '0:8008/tcp', '--network=boxel'); - } else { - dockerParams.push( - `--ip=${synCfg.host}`, - '-p', - `${synCfg.port}:8008/tcp`, - '--network=boxel', + let hostPort = SYNAPSE_PORT; + let synCfg!: SynapseConfig; + let containerName!: string; + let synapseId!: string; + let attempts = useDynamicHostPort ? dynamicHostPortStartAttempts : 1; + + for (let attempt = 1; attempt <= attempts; attempt++) { + hostPort = useDynamicHostPort ? await findAvailablePort() : SYNAPSE_PORT; + synCfg = await cfgDirFromTemplate(opts?.template ?? 'test', opts?.dataDir, { + host: useDynamicHostPort ? '127.0.0.1' : SYNAPSE_IP_ADDRESS, + port: hostPort, + publicBaseUrl: `http://localhost:${hostPort}`, + }); + containerName = + opts?.containerName || + (isEnvironmentMode() + ? getSynapseContainerName() + : path.basename(synCfg.configDir)); + console.log( + `Starting synapse with config dir ${synCfg.configDir} in container ${containerName}...`, ); - } - const synapseId = await dockerRun({ - image: 'matrixdotorg/synapse:v1.126.0', - containerName, - dockerParams, - applicationParams: ['run'], - runAsUser: true, - }); + let dockerParams: string[] = [ + '--rm', + '-v', + `${synCfg.configDir}:/data`, + '-v', + `${path.join(__dirname, 'templates')}:/custom/templates/`, + ]; + if (useDynamicHostPort) { + // In dynamic-host-port mode multiple harnesses may run concurrently, so + // we must not claim the shared fixed Synapse container IP. + dockerParams.push('-p', `${hostPort}:8008/tcp`, '--network=boxel'); + } else { + dockerParams.push( + `--ip=${synCfg.host}`, + '-p', + `${synCfg.port}:8008/tcp`, + '--network=boxel', + ); + } - console.log(`Started synapse with id ${synapseId} on port ${synCfg.port}`); + try { + synapseId = await dockerRun({ + image: 'matrixdotorg/synapse:v1.126.0', + containerName, + dockerParams, + applicationParams: ['run'], + runAsUser: true, + }); + break; + } catch (error) { + if ( + !useDynamicHostPort || + !isPortBindError(error) || + attempt === attempts + ) { + throw error; + } + console.warn( + `Synapse host port ${hostPort} was claimed before Docker bound it; retrying (${attempt}/${attempts})...`, + ); + if (!opts?.dataDir) { + await fse.remove(synCfg.configDir); + } + } + } + + console.log(`Started synapse with id ${synapseId} on port ${hostPort}`); // Await Synapse healthcheck await dockerExec({ @@ -199,9 +275,13 @@ export async function synapseStart( ], }); - let hostPort = synCfg.port; - if (isEnvironmentMode() || opts?.dynamicHostPort) { - hostPort = await resolveHostPort(synapseId); + if (useDynamicHostPort) { + let resolvedPort = await resolveHostPort(synapseId); + if (resolvedPort !== hostPort) { + throw new Error( + `Synapse started on unexpected host port ${resolvedPort}; expected ${hostPort}`, + ); + } console.log(`Synapse dynamic host port: ${hostPort}`); } diff --git a/packages/realm-server/scripts/start-production.sh b/packages/realm-server/scripts/start-production.sh index 139f958931..f9e9e42ac6 100755 --- a/packages/realm-server/scripts/start-production.sh +++ b/packages/realm-server/scripts/start-production.sh @@ -25,6 +25,7 @@ EXTERNAL_CATALOG_REALM_URL="${RESOLVED_EXTERNAL_CATALOG_REALM_URL:-$DEFAULT_EXTE NODE_NO_WARNINGS=1 \ LOW_CREDIT_THRESHOLD=2000 \ MATRIX_URL=https://matrix.boxel.ai \ + MATRIX_SERVER_NAME=boxel.ai \ BOXEL_HOST_URL=https://app.boxel.ai \ REALM_SERVER_MATRIX_USERNAME=realm_server \ PUBLISHED_REALM_BOXEL_SPACE_DOMAIN='boxel.space' \ diff --git a/packages/realm-server/scripts/start-staging.sh b/packages/realm-server/scripts/start-staging.sh index 207381bb6a..51d4ed1c75 100755 --- a/packages/realm-server/scripts/start-staging.sh +++ b/packages/realm-server/scripts/start-staging.sh @@ -25,6 +25,7 @@ EXTERNAL_CATALOG_REALM_URL="${RESOLVED_EXTERNAL_CATALOG_REALM_URL:-$DEFAULT_EXTE NODE_NO_WARNINGS=1 \ LOW_CREDIT_THRESHOLD=2000 \ MATRIX_URL=https://matrix-staging.stack.cards \ + MATRIX_SERVER_NAME=stack.cards \ BOXEL_HOST_URL=https://realms-staging.stack.cards \ REALM_SERVER_MATRIX_USERNAME=realm_server \ PUBLISHED_REALM_BOXEL_SPACE_DOMAIN='staging.boxel.dev' \ diff --git a/packages/realm-server/server.ts b/packages/realm-server/server.ts index 4a08890d43..53a251f3c7 100644 --- a/packages/realm-server/server.ts +++ b/packages/realm-server/server.ts @@ -707,7 +707,9 @@ export class RealmServer { hostsOwnAssets: false, assetsURL: this.assetsURL.href, matrixURL: this.matrixClient.matrixURL.href.replace(/\/$/, ''), - matrixServerName: this.matrixClient.matrixURL.hostname, + matrixServerName: + process.env.MATRIX_SERVER_NAME || + this.matrixClient.matrixURL.hostname, realmServerURL: this.serverURL.href, resolvedBaseRealmURL: rewriteRealmURL(config.resolvedBaseRealmURL), resolvedCatalogRealmURL: rewriteRealmURL( diff --git a/packages/realm-server/tests/index.ts b/packages/realm-server/tests/index.ts index d4b8302424..61e5f187df 100644 --- a/packages/realm-server/tests/index.ts +++ b/packages/realm-server/tests/index.ts @@ -158,6 +158,7 @@ import './server-endpoints/queue-status-test'; import './server-endpoints/realm-lifecycle-test'; import './server-endpoints/search-test'; import './server-endpoints/search-prerendered-test'; +import './server-config-test'; import './server-endpoints/info-test'; import './server-endpoints/stripe-session-test'; import './server-endpoints/stripe-webhook-test'; diff --git a/packages/realm-server/tests/server-config-test.ts b/packages/realm-server/tests/server-config-test.ts new file mode 100644 index 0000000000..81ce96e989 --- /dev/null +++ b/packages/realm-server/tests/server-config-test.ts @@ -0,0 +1,65 @@ +import { module, test } from 'qunit'; +import { basename } from 'path'; +import { dirSync } from 'tmp'; + +import { RealmServer } from '../server'; + +module(basename(__filename), function () { + test('prefers MATRIX_SERVER_NAME over matrix URL hostname in host config', async function (assert) { + let originalMatrixServerName = process.env.MATRIX_SERVER_NAME; + let tempDir = dirSync({ unsafeCleanup: true }); + + process.env.MATRIX_SERVER_NAME = 'stack.cards'; + + try { + let server = new RealmServer({ + serverURL: new URL('http://127.0.0.1:4448'), + realms: [], + virtualNetwork: {} as any, + matrixClient: { + matrixURL: new URL('http://localhost:8008/'), + } as any, + realmServerSecretSeed: 'test-realm-server-secret', + realmSecretSeed: 'test-realm-secret', + grafanaSecret: 'test-grafana-secret', + realmsRootPath: tempDir.name, + dbAdapter: {} as any, + queue: {} as any, + definitionLookup: {} as any, + assetsURL: new URL('http://example.com/notional-assets-host/'), + matrixRegistrationSecret: 'test-matrix-registration-secret', + getIndexHTML: async () => + ``, + }); + + let html = await (server as any).retrieveIndexHTML(); + let match = html.match( + //, + ); + + assert.ok(match, 'host config environment meta tag is present'); + + let config = JSON.parse(decodeURIComponent(match![1])); + assert.strictEqual( + config.matrixServerName, + 'stack.cards', + 'uses MATRIX_SERVER_NAME override in host config', + ); + } finally { + if (originalMatrixServerName == null) { + delete process.env.MATRIX_SERVER_NAME; + } else { + process.env.MATRIX_SERVER_NAME = originalMatrixServerName; + } + tempDir.removeCallback(); + } + }); +}); diff --git a/packages/software-factory/src/harness/isolated-realm-stack.ts b/packages/software-factory/src/harness/isolated-realm-stack.ts index 7e634025c8..828d2c62c2 100644 --- a/packages/software-factory/src/harness/isolated-realm-stack.ts +++ b/packages/software-factory/src/harness/isolated-realm-stack.ts @@ -341,6 +341,7 @@ export async function startIsolatedRealmStack({ GRAFANA_SECRET, HOST_URL: context.hostURL, MATRIX_URL: context.matrixURL, + MATRIX_SERVER_NAME: new URL(context.matrixURL).hostname, MATRIX_REGISTRATION_SHARED_SECRET: context.matrixRegistrationSecret, REALM_SERVER_MATRIX_USERNAME: DEFAULT_MATRIX_SERVER_USERNAME, REALM_SERVER_FULL_INDEX_ON_STARTUP: String(fullIndexOnStartup), diff --git a/packages/software-factory/src/harness/shared.ts b/packages/software-factory/src/harness/shared.ts index 3710f3f43c..2b8034c442 100644 --- a/packages/software-factory/src/harness/shared.ts +++ b/packages/software-factory/src/harness/shared.ts @@ -781,19 +781,53 @@ export async function stopManagedProcess(proc: SpawnedProcess): Promise { if (proc.exitCode !== null) { return; } - let stopped = new Promise((resolve) => { + let stopped = new Promise<'stopped'>((resolve) => { let onMessage = (message: unknown) => { if (message === 'stopped') { proc.off('message', onMessage); - resolve(); + resolve('stopped'); } }; proc.on('message', onMessage); }); + let exited = new Promise<'exited'>((resolve) => { + let onExit = () => { + proc.off('exit', onExit); + proc.off('error', onExit); + resolve('exited'); + }; + proc.on('exit', onExit); + proc.on('error', onExit); + }); proc.send('stop'); - await Promise.race([ + let stopResult = await Promise.race([ stopped, - new Promise((resolve) => setTimeout(resolve, 15_000)), + exited, + new Promise((resolve) => setTimeout(() => resolve(false), 15_000)), ]); - proc.send('kill'); + if (stopResult === false && proc.exitCode === null) { + proc.send('kill'); + } + if (proc.exitCode === null) { + let exitResult = await Promise.race([ + exited, + new Promise((resolve) => setTimeout(() => resolve(false), 15_000)), + ]); + if (exitResult === false && proc.exitCode === null) { + try { + proc.kill(); + } catch { + // best effort hard-kill + } + exitResult = await Promise.race([ + exited, + new Promise((resolve) => + setTimeout(() => resolve(false), 5_000), + ), + ]); + if (exitResult === false && proc.exitCode === null) { + throw new Error('Failed to stop managed process within timeout'); + } + } + } }