Add tokenization regression tests

.github/workflows/node.js.yml

@@ -0,0 +1,17 @@
+name: Node.js CI
+on:
+  push:
+    branches: [ master, work ]
+  pull_request:
+    branches: [ master, work ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '18'
+    - run: npm ci
+    - run: npm run test

.gitignore

@@ -2,3 +2,4 @@
 .vs/
 bin/
 obj/
+node_modules/

.vscodeignore

@@ -1,5 +1,6 @@
 Pluto.tmbundle/info.plist
 Pluto.tmbundle/Snippets/**.tmSnippet
+node_modules/
 .gitignore
 *.sublime-build
 visual-check.*

package.json

@@ -2,11 +2,17 @@
 	"name": "pluto-syntax-highlighting",
 	"displayName": "Pluto Syntax Highlighting",
 	"description": "Provides syntax highlighting and snippets for Pluto, a Lua dialect.",
-	"keywords": ["Pluto language"],
+	"keywords": [
+		"Pluto language"
+	],
 	"version": "0.7.0",
 	"publisher": "calamity-inc",
 	"icon": "icon.png",
-	"repository": "https://github.com/PlutoLang/Syntax-Highlighting",
+	"repository": "https://github.com/PlutoLang/Syntax-Highlighting",
+        "scripts": {
+                "generate-baseline": "node scripts/generate-baseline.js",
+                "test": "node scripts/test-tokenization.js"
+        },
 	"engines": {
 		"vscode": "^1.65.0"
 	},
@@ -17,8 +23,12 @@
 		"languages": [
 			{
 				"id": "pluto",
-				"aliases": ["Pluto"],
-				"extensions": [".pluto"],
+				"aliases": [
+					"Pluto"
+				],
+				"extensions": [
+					".pluto"
+				],
 				"configuration": "./language-config.json",
 				"icon": {
 					"light": "icon.png",
@@ -54,5 +64,9 @@
 	"activationEvents": [
 		"onCommand:pluto-syntax-highlighting.run",
 		"onCommand:pluto-syntax-highlighting.lint"
-	]
+	],
+	"devDependencies": {
+		"vscode-oniguruma": "^2.0.1",
+		"vscode-textmate": "^9.2.0"
+	}
 }

scripts/generate-baseline.js

@@ -0,0 +1,59 @@
+const fs = require('fs');
+const path = require('path');
+const vsctm = require('vscode-textmate');
+const onig = require('vscode-oniguruma');
+
+async function loadGrammar() {
+    const wasmPath = require.resolve('vscode-oniguruma/release/onig.wasm');
+    const wasm = fs.readFileSync(wasmPath).buffer;
+    await onig.loadWASM(wasm);
+    const onigLib = {
+        createOnigScanner: (sources) => new onig.OnigScanner(sources),
+        createOnigString: (str) => new onig.OnigString(str)
+    };
+
+    const registry = new vsctm.Registry({
+        onigLib,
+        loadGrammar: (scopeName) => {
+            if (scopeName === 'source.pluto') {
+                const tmLanguage = fs.readFileSync(path.join(__dirname, '..', 'Pluto.tmbundle', 'Syntaxes', 'Pluto.tmLanguage'), 'utf8');
+                return vsctm.parseRawGrammar(tmLanguage);
+            }
+            return null;
+        }
+    });
+
+    return registry.loadGrammar('source.pluto');
+}
+
+async function tokenizeFile(grammar, filePath) {
+    const lines = fs.readFileSync(filePath, 'utf8').split(/\r?\n/);
+    const result = [];
+    let ruleStack = null;
+    for (const line of lines) {
+        const res = grammar.tokenizeLine(line, ruleStack);
+        result.push(res.tokens.map(t => ({
+            startIndex: t.startIndex,
+            endIndex: t.endIndex,
+            scopes: t.scopes
+        })));
+        ruleStack = res.ruleStack;
+    }
+    return result;
+}
+
+async function main() {
+    const grammar = await loadGrammar();
+    const files = fs.readdirSync('.').filter(f => /^visual-.*\.pluto$/.test(f));
+    for (const file of files) {
+        const tokens = await tokenizeFile(grammar, file);
+        const outPath = path.join('tests', 'baseline', file + '.json');
+        fs.writeFileSync(outPath, JSON.stringify(tokens, null, 2));
+        console.log(`Wrote ${outPath}`);
+    }
+}
+
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});

scripts/test-tokenization.js

@@ -0,0 +1,67 @@
+const fs = require('fs');
+const path = require('path');
+const vsctm = require('vscode-textmate');
+const onig = require('vscode-oniguruma');
+
+async function loadGrammar() {
+    const wasmPath = require.resolve('vscode-oniguruma/release/onig.wasm');
+    const wasm = fs.readFileSync(wasmPath).buffer;
+    await onig.loadWASM(wasm);
+    const onigLib = {
+        createOnigScanner: (sources) => new onig.OnigScanner(sources),
+        createOnigString: (str) => new onig.OnigString(str)
+    };
+
+    const registry = new vsctm.Registry({
+        onigLib,
+        loadGrammar: (scopeName) => {
+            if (scopeName === 'source.pluto') {
+                const tmLanguage = fs.readFileSync(path.join(__dirname, '..', 'Pluto.tmbundle', 'Syntaxes', 'Pluto.tmLanguage'), 'utf8');
+                return vsctm.parseRawGrammar(tmLanguage);
+            }
+            return null;
+        }
+    });
+
+    return registry.loadGrammar('source.pluto');
+}
+
+async function tokenizeFile(grammar, filePath) {
+    const lines = fs.readFileSync(filePath, 'utf8').split(/\r?\n/);
+    const result = [];
+    let ruleStack = null;
+    for (const line of lines) {
+        const res = grammar.tokenizeLine(line, ruleStack);
+        result.push(res.tokens.map(t => ({
+            startIndex: t.startIndex,
+            endIndex: t.endIndex,
+            scopes: t.scopes
+        })));
+        ruleStack = res.ruleStack;
+    }
+    return result;
+}
+
+async function main() {
+    const grammar = await loadGrammar();
+    const files = fs.readdirSync('.').filter(f => /^visual-.*\.pluto$/.test(f));
+    let ok = true;
+    for (const file of files) {
+        const tokens = await tokenizeFile(grammar, file);
+        const baseline = JSON.parse(fs.readFileSync(path.join('tests', 'baseline', file + '.json'), 'utf8'));
+        if (JSON.stringify(tokens) !== JSON.stringify(baseline)) {
+            console.error(`Tokenization mismatch for ${file}`);
+            ok = false;
+        } else {
+            console.log(`${file}: ok`);
+        }
+    }
+    if (!ok) {
+        process.exit(1);
+    }
+}
+
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});