From 80c6cbd1912446aa39d663a82f5e1e89dd65d4fe Mon Sep 17 00:00:00 2001 From: AnHeuermann <38031952+AnHeuermann@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:26:02 +0200 Subject: [PATCH] Make e2e tests deterministic instead of sleeping The language server initializes and parses documents asynchronously, so waiting a fixed amount of time after activation was racy: slow machines (CI) could still see empty provider results while fast machines wasted seconds. - Add executeProviderUntilResult() helper that polls a provider command until it returns a non-empty result (or times out) - Use it in the gotoDeclaration, mslLibrary, and symbolinformation tests - Drop the fixed sleep(5000) from activate() and document that the server is not guaranteed ready when it returns - Pin the VS Code test version to 'stable' and cache the download in CI for reproducible, faster runs - Ignore *.tsbuildinfo Co-Authored-By: Claude Opus 4.8 --- .github/workflows/test.yml | 6 +++ .gitignore | 1 + client/src/test/gotoDeclaration.test.ts | 7 ++-- client/src/test/helper.ts | 45 +++++++++++++++++++++-- client/src/test/mslLibrary.test.ts | 7 ++-- client/src/test/runTest.ts | 1 + client/src/test/symbolinformation.test.ts | 6 +-- 7 files changed, 58 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e7aa11..295b4fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,12 @@ jobs: node-version: 24 registry-url: https://registry.npmjs.org/ + - name: Cache VS Code test downloads + uses: actions/cache@v4 + with: + path: .vscode-test + key: vscode-test-${{ runner.os }}-stable + - name: Install X server run: | sudo apt-get update diff --git a/.gitignore b/.gitignore index 2a35d75..49c207d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ modelica-language-server*.vsix node_modules/ out/ +*.tsbuildinfo client/testFixture/.vscode/settings.json diff --git a/client/src/test/gotoDeclaration.test.ts b/client/src/test/gotoDeclaration.test.ts index bdda157..ad0148c 100644 --- a/client/src/test/gotoDeclaration.test.ts +++ b/client/src/test/gotoDeclaration.test.ts @@ -35,7 +35,7 @@ import * as vscode from 'vscode'; import * as assert from 'assert'; -import { getDocUri, activate } from './helper'; +import { getDocUri, activate, executeProviderUntilResult } from './helper'; suite('Goto Declaration', () => { test('onDeclaration()', async () => { @@ -43,10 +43,9 @@ suite('Goto Declaration', () => { await activate(docUri); const position = new vscode.Position(4, 18); - const actualLocations = await vscode.commands.executeCommand( + const actualLocations = await executeProviderUntilResult( 'vscode.executeDeclarationProvider', - docUri, - position, + [docUri, position], ); assert.strictEqual(actualLocations.length, 1); diff --git a/client/src/test/helper.ts b/client/src/test/helper.ts index 2b0a785..507f5f5 100644 --- a/client/src/test/helper.ts +++ b/client/src/test/helper.ts @@ -41,8 +41,19 @@ export let editor: vscode.TextEditor; export let documentEol: string; export let platformEol: string; + /** - * Activates the OpenModelica.modelica-language-server extension + * Activates the OpenModelica.modelica-language-server extension and opens the + * given document. + * + * This resolves once the extension is activated and the document is open, but + * NOT once the language server has finished initializing and parsing it (that + * happens asynchronously). Callers must therefore not assume the server is + * ready when this returns: run provider commands through + * {@link executeProviderUntilResult}, which polls until a non-empty result is + * available instead of relying on a fixed delay. + * + * @param docUri The document to open in the editor. */ export async function activate(docUri: vscode.Uri): Promise { // The extensionId is `publisher.name` from package.json @@ -54,14 +65,40 @@ export async function activate(docUri: vscode.Uri): Promise { try { doc = await vscode.workspace.openTextDocument(docUri); editor = await vscode.window.showTextDocument(doc); - await sleep(5000); // Wait for server activation + // No fixed wait for server activation here: callers use + // `executeProviderUntilResult`, which polls until the server is ready. } catch (e) { console.error(e); } } -async function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); +/** + * Repeatedly run a VS Code command until it returns a non-empty result. + * + * The language server initializes and parses documents asynchronously, so a + * single fixed delay after activation is racy: on slower machines (e.g. CI) + * the provider can still return an empty result. Polling avoids that flakiness + * while keeping fast machines fast. + * + * @param command Command id to execute, e.g. `vscode.executeDeclarationProvider`. + * @param args Arguments forwarded to the command. + * @param timeoutMs Maximum time to keep retrying. + * @param intervalMs Delay between attempts. + * @returns The first non-empty result, or the last (empty) result on timeout. + */ +export async function executeProviderUntilResult( + command: string, + args: unknown[], + timeoutMs = 20000, + intervalMs = 250, +): Promise { + const deadline = Date.now() + timeoutMs; + let result = await vscode.commands.executeCommand(command, ...args); + while ((!result || result.length === 0) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + result = await vscode.commands.executeCommand(command, ...args); + } + return result; } export const getDocPath = (p: string): string => { diff --git a/client/src/test/mslLibrary.test.ts b/client/src/test/mslLibrary.test.ts index 59a7785..e457b97 100644 --- a/client/src/test/mslLibrary.test.ts +++ b/client/src/test/mslLibrary.test.ts @@ -35,7 +35,7 @@ import * as vscode from 'vscode'; import * as assert from 'assert'; -import { getDocUri, activate } from './helper'; +import { getDocUri, activate, executeProviderUntilResult } from './helper'; suite('MSL Library Support', () => { test('go-to-declaration resolves MSL type into MSL directory', async () => { @@ -51,10 +51,9 @@ suite('MSL Library Support', () => { // Line 2: " Modelica.Units.SI.Voltage v;" — cursor on "Modelica" at column 4 const position = new vscode.Position(2, 4); - const actualLocations = await vscode.commands.executeCommand( + const actualLocations = await executeProviderUntilResult( 'vscode.executeDeclarationProvider', - docUri, - position, + [docUri, position], ); assert.ok(actualLocations.length > 0, 'Expected at least one declaration location'); diff --git a/client/src/test/runTest.ts b/client/src/test/runTest.ts index a466f28..c7461de 100644 --- a/client/src/test/runTest.ts +++ b/client/src/test/runTest.ts @@ -72,6 +72,7 @@ async function main() { // Download VS Code, unzip it and run the integration test await runTests({ + version: 'stable', extensionDevelopmentPath, extensionTestsPath, launchArgs: [testFixturePath], diff --git a/client/src/test/symbolinformation.test.ts b/client/src/test/symbolinformation.test.ts index 8a7adfd..16782b8 100644 --- a/client/src/test/symbolinformation.test.ts +++ b/client/src/test/symbolinformation.test.ts @@ -35,7 +35,7 @@ import * as vscode from 'vscode'; import * as assert from 'assert'; -import { getDocUri, activate } from './helper'; +import { getDocUri, activate, executeProviderUntilResult } from './helper'; suite('Symbol Information', () => { const docUri = getDocUri('MyLibrary.mo'); @@ -71,9 +71,9 @@ async function testSymbolInformation( await activate(docUri); // Execute `vscode.executeDocumentSymbolProvider` to get file outline - const actualSymbolInformation = await vscode.commands.executeCommand( + const actualSymbolInformation = await executeProviderUntilResult( 'vscode.executeDocumentSymbolProvider', - docUri, + [docUri], ); assertDocumentSymbolsEqual(expectedDocumentSymbols, actualSymbolInformation);