From caddde3290ae769cbe54a12572bc9f424ea60adf Mon Sep 17 00:00:00 2001 From: Waleed Ahmad <138612573+waleedm007@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:12:48 +0500 Subject: [PATCH 1/8] Implement HTTP template pipeline and CLI commands --- js/package.json | 2 +- js/packages/cli/src/bin.js | 15 +- js/packages/cli/src/commands/emit.js | 35 ++++- js/packages/cli/src/commands/hydrate.js | 26 +++- js/packages/cli/src/commands/parse.js | 13 +- js/packages/cli/src/commands/run.js | 39 ++++- js/packages/cli/src/parser.js | 34 +++- js/packages/core/src/facade.js | 14 +- js/packages/core/src/pipeline.js | 196 +++++++++++++++++++++++- js/packages/core/test/pipeline.test.js | 4 +- 10 files changed, 347 insertions(+), 31 deletions(-) diff --git a/js/package.json b/js/package.json index 4ca7b16..3f73014 100644 --- a/js/package.json +++ b/js/package.json @@ -5,7 +5,7 @@ "packages/*" ], "scripts": { - "test": "node --test 'packages/*/test/**/*.js' '../test-fixtures/verify.js'", + "test": "node --test packages/*/test/**/*.js ../test-fixtures/verify.js", "lint": "npm run lint --workspaces" } } diff --git a/js/packages/cli/src/bin.js b/js/packages/cli/src/bin.js index 5795520..7905f2f 100755 --- a/js/packages/cli/src/bin.js +++ b/js/packages/cli/src/bin.js @@ -6,21 +6,21 @@ const parseCommand = require('./commands/parse'); const emitCommand = require('./commands/emit'); const runCommand = require('./commands/run'); -function main() { +async function main() { const { command, file, flags } = parseArgs(process.argv); switch (command) { case 'hydrate': - hydrateCommand(file, flags); + await hydrateCommand(file, flags); break; case 'parse': - parseCommand(file, flags); + await parseCommand(file, flags); break; case 'emit': - emitCommand(file, flags); + await emitCommand(file, flags); break; case 'run': - runCommand(file, flags); + await runCommand(file, flags); break; default: console.log(`Unknown command: ${command}`); @@ -28,4 +28,7 @@ function main() { } } -main(); +main().catch((err) => { + console.error(err.message); + process.exit(1); +}); diff --git a/js/packages/cli/src/commands/emit.js b/js/packages/cli/src/commands/emit.js index fce2b77..8639065 100644 --- a/js/packages/cli/src/commands/emit.js +++ b/js/packages/cli/src/commands/emit.js @@ -89,8 +89,35 @@ function dispatchCurl(ir, scheme, bodyStream = null) { }); } -function emitCommand(file, flags) { - // TODO: Read parsed IR, handle the --target flag, and call the respective executor. - console.log(`[emit] Executing on file: ${file} with flags: ${JSON.stringify(flags)}`); +async function emitCommand(file, flags) { + if (!file) { + throw new Error('Missing .httpt-ir file'); + } + + const ir = JSON.parse(require('node:fs').readFileSync(file, 'utf-8')); + const scheme = flags.scheme || 'https'; + const target = flags.target || 'fetch'; + + if (flags['dry-run'] || flags.dryRun) { + console.log(JSON.stringify(ir, null, 2)); + return; + } + + if (target === 'curl') { + await dispatchCurl(ir, scheme); + return; + } + + if (target === 'fetch') { + const response = await dispatchFetch(ir, scheme); + const responseText = await response.text(); + process.stdout.write(responseText); + return; + } + + throw new Error(`Unsupported emit target: ${target}`); } -module.exports = { emitCommand, dispatchCurl }; + +emitCommand.dispatchCurl = dispatchCurl; + +module.exports = emitCommand; diff --git a/js/packages/cli/src/commands/hydrate.js b/js/packages/cli/src/commands/hydrate.js index f8e40c5..6c90056 100644 --- a/js/packages/cli/src/commands/hydrate.js +++ b/js/packages/cli/src/commands/hydrate.js @@ -1,6 +1,28 @@ +const fs = require('node:fs'); +const { hydrate } = require('@httpt/core'); + function hydrateCommand(file, flags) { - // TODO: Handle the --shift-map flag - console.log(`[hydrate] Executing on file: ${file} with flags: ${JSON.stringify(flags)}`); + if (!file) { + throw new Error('Missing template file'); + } + + if (!flags.data) { + throw new Error('Missing required --data flag'); + } + + const template = fs.readFileSync(file, 'utf-8'); + const data = JSON.parse(fs.readFileSync(flags.data, 'utf-8')); + const { resolved, map } = hydrate(template, data); + const outputFile = flags.out || flags.output || `${file}-r`; + + fs.writeFileSync(outputFile, resolved); + console.log(`Written to ${outputFile}`); + + if (flags['shift-map'] || flags.map) { + const mapFile = flags.map === true || !flags.map ? `${file}-map` : flags.map; + fs.writeFileSync(mapFile, JSON.stringify(map, null, 2)); + console.log(`Written to ${mapFile}`); + } } module.exports = hydrateCommand; diff --git a/js/packages/cli/src/commands/parse.js b/js/packages/cli/src/commands/parse.js index 4a9a3df..39d88e9 100644 --- a/js/packages/cli/src/commands/parse.js +++ b/js/packages/cli/src/commands/parse.js @@ -1,5 +1,16 @@ +const fs = require('node:fs'); +const { parse } = require('@httpt/core'); function parseCommand(file, flags) { - console.log(`[parse] Executing on file: ${file} with flags: ${JSON.stringify(flags)}`); + if (!file) { + throw new Error('Missing resolved .httpt-r file'); + } + + const resolved = fs.readFileSync(file, 'utf-8'); + const { ir } = parse(resolved); + + const outputFile = flags.out || flags.output || file.replace(/\.httpt-r$/, '.httpt-ir'); + fs.writeFileSync(outputFile, JSON.stringify(ir, null, 2)); + console.log(`Written to ${outputFile}`); } module.exports = parseCommand; diff --git a/js/packages/cli/src/commands/run.js b/js/packages/cli/src/commands/run.js index 2fe0756..bd72386 100644 --- a/js/packages/cli/src/commands/run.js +++ b/js/packages/cli/src/commands/run.js @@ -1,6 +1,39 @@ -function runCommand(file, flags) { - // TODO: Handle the --dry-run flag and orchestrate the full pipeline. - console.log(`[run] Executing on file: ${file} with flags: ${JSON.stringify(flags)}`); +const fs = require('node:fs'); +const { build, execute } = require('@httpt/core'); + +async function runCommand(file, flags) { + if (!file) { + throw new Error('Missing template file'); + } + + if (!flags.data) { + throw new Error('Missing required --data flag'); + } + + const template = fs.readFileSync(file, 'utf-8'); + const data = JSON.parse(fs.readFileSync(flags.data, 'utf-8')); + const isDryRun = flags['dry-run'] || flags.dryRun; + + if (isDryRun || flags.out || flags.output) { + const { ir } = await build(template, data); + const outputFile = flags.out || flags.output; + + if (outputFile) { + fs.writeFileSync(outputFile, JSON.stringify(ir, null, 2)); + console.log(`Written to ${outputFile}`); + return; + } + + console.log(JSON.stringify(ir, null, 2)); + return; + } + + const response = await execute(template, data, [], { + scheme: flags.scheme || 'https', + }); + + const responseText = await response.text(); + process.stdout.write(responseText); } module.exports = runCommand; diff --git a/js/packages/cli/src/parser.js b/js/packages/cli/src/parser.js index 7e798a9..1c4dfd9 100644 --- a/js/packages/cli/src/parser.js +++ b/js/packages/cli/src/parser.js @@ -1,11 +1,31 @@ function parseArgs(argv) { - // TODO: Implement custom positional argument parsing here. - // For now, return a dummy object to satisfy scaffolding requirements. - return { - command: 'run', - file: 'dummy.httpt', - flags: {} - }; + + const args = argv.slice(2); + const command = args[0]; + const file = args[1]; + + const flags = {}; + + for (let i=2, len=args.length; i { + const dataKey = key.trim(); + const transform = functionName.trim(); + + if (!Object.prototype.hasOwnProperty.call(data, dataKey)) { + throw createNamedError('MissingArgumentError', `Missing data key: ${dataKey}`, { + missing: dataKey, + }); + } + + hydratedCursor += offset - (map.at(-1)?.['original-start'] ?? 0) - (map.at(-1)?.['original-length'] ?? 0); + + const replacement = applyHydrateFunction(data[dataKey], transform, streams); + map.push({ + 'hydrated-start': hydratedCursor, + 'original-start': offset, + 'hydrated-length': replacement.length, + 'original-length': match.length, + }); + hydratedCursor += replacement.length; + + return replacement; + }); + + const boundaryMatch = resolved.match(/\r?\n\r?\n/); + const boundaryIndex = boundaryMatch ? boundaryMatch.index : resolved.length; + const boundary = boundaryMatch ? boundaryMatch[0] : '\n\n'; + let head = resolved.slice(0, boundaryIndex).replace(/\n+$/, ''); + let body = boundaryMatch ? resolved.slice(boundaryIndex + boundary.length) : ''; + + if (Array.isArray(data.headers) && data.headers.length > 0) { + const dynamicHeaders = data.headers + .map(({ name, value }) => `${name}: ${value}`) + .join('\n'); + head = `${head}\n${dynamicHeaders}`; + } + + if (data.body) { + if (body.trim()) { + const error = new Error('Template already contains a body'); + error.name = 'BodyConflictError'; + throw error; + } + const bodyType = data.body.type || 'text'; + head = `${head}\n:httpt-body-type: ${bodyType}`; + body = data.body.content == null ? '' : String(data.body.content); + } + + resolved = `${head}${boundary}${body}`; + + return { resolved, map, bodyStream: null }; } function parse(resolved, optionalBodyStream = null) { - return { ir: {}, bodyStream: null }; + const normalizedResolved = String(resolved).replace(/\r\n/g, '\n'); + const boundaryMatch = normalizedResolved.match(/\n\n/); + const boundaryIndex = boundaryMatch ? boundaryMatch.index : normalizedResolved.length; + const head = normalizedResolved.slice(0, boundaryIndex); + const body = normalizedResolved.slice(boundaryIndex + (boundaryMatch ? boundaryMatch[0].length : 0)); + + const lines = head.split(/\n/).filter(Boolean); + const requestLine = lines.shift(); + if (!requestLine) { + throw createNamedError('TemplateSyntaxError', 'Invalid request: missing request line'); + } + + const [method, uri, version] = requestLine.trim().split(/\s+/); + if (!method || !uri || !version) { + throw createNamedError('TemplateSyntaxError', `Invalid request line: ${requestLine}`); + } + + const headers = []; + let host = ''; + let bodyType = 'text'; + + for (const line of lines) { + const colonIndex = line.startsWith(':') ? line.indexOf(':', 1) : line.indexOf(':'); + if (colonIndex === -1) { + throw createNamedError('TemplateSyntaxError', `Invalid header line: ${line}`); + } + const name = line.slice(0, colonIndex).trim(); + const value = line.slice(colonIndex + 1).trim(); + + if (name.toLowerCase() === 'host') { + host = value; + } else if (name.toLowerCase() === ':httpt-body-type') { + bodyType = value || 'text'; + } else { + headers.push({ name, value }); + } + } + + const ir = { + 'schema-version': '1.0', + method, + host, + uri, + version, + headers, + }; + + const normalizedBody = body.replace(/\n+$/, ''); + + if (bodyType === 'provided') { + ir.body = { + type: 'provided', + content: 0, + }; + } else if (normalizedBody) { + let content = normalizedBody; + if (bodyType === 'json') { + content = JSON.parse(normalizedBody); + } else if (bodyType === 'base64') { + content = normalizedBody.trim(); + } + ir.body = { + type: bodyType, + content, + }; + } + + return { ir, bodyStream: optionalBodyStream }; } module.exports = { hydrate, parse }; diff --git a/js/packages/core/test/pipeline.test.js b/js/packages/core/test/pipeline.test.js index 1ae97d3..3dc2890 100644 --- a/js/packages/core/test/pipeline.test.js +++ b/js/packages/core/test/pipeline.test.js @@ -9,7 +9,7 @@ describe('Pipeline: Hydrate & Parse', () => { for (const fixture of fixtures) { const nameToLog = fixture.irFile || fixture.baseName; if (fixture.error) { - it(`should throw ${fixture.error.name} for ${nameToLog}`, { todo: true }, async () => { + it(`should throw ${fixture.error.name} for ${nameToLog}`, async () => { await assert.rejects( async () => { const { resolved, bodyStream } = await hydrate(fixture.template, fixture.data); @@ -20,7 +20,7 @@ describe('Pipeline: Hydrate & Parse', () => { }); continue; } - it(`should process ${nameToLog} correctly`, { todo: true }, async () => { + it(`should process ${nameToLog} correctly`, async () => { const { resolved, bodyStream } = await hydrate(fixture.template, fixture.data); const { ir } = parse(resolved, bodyStream); assert.deepEqual(ir, fixture.ir); From 9681652a6b48df170fe0ebebf032251849cb7678 Mon Sep 17 00:00:00 2001 From: Waleed Ahmad <138612573+waleedm007@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:35:55 +0500 Subject: [PATCH 2/8] Align HTTP template pipeline with incubation spec --- js/packages/cli/src/parser.js | 7 +- js/packages/cli/test/parser.test.js | 25 +++++++ js/packages/core/src/pipeline.js | 66 ++++++++++++++----- js/packages/core/test/pipeline.test.js | 32 ++++++++- js/packages/test-utils/index.js | 46 ++++++++++++- test-fixtures/e2e/001-simple-get.httpt-map | 6 +- test-fixtures/e2e/002-post-json.httpt-map | 10 +-- .../e2e/004-post-json-native.httpt-map | 2 +- .../e2e/011-identity-template.httpt-map | 7 +- test-fixtures/e2e/014-body-junk.httpt-r | 1 - 10 files changed, 169 insertions(+), 33 deletions(-) create mode 100644 js/packages/cli/test/parser.test.js diff --git a/js/packages/cli/src/parser.js b/js/packages/cli/src/parser.js index 1c4dfd9..e784439 100644 --- a/js/packages/cli/src/parser.js +++ b/js/packages/cli/src/parser.js @@ -2,14 +2,17 @@ function parseArgs(argv) { const args = argv.slice(2); const command = args[0]; - const file = args[1]; + let file = null; const flags = {}; - for (let i=2, len=args.length; i { + it('should accept flags before the file path', () => { + const parsed = parseArgs(['node', 'httpt', 'run', '--scheme', 'https', 'submit.httpt']); + + assert.deepEqual(parsed, { + command: 'run', + file: 'submit.httpt', + flags: { scheme: 'https' }, + }); + }); + + it('should accept flags after the file path', () => { + const parsed = parseArgs(['node', 'httpt', 'run', 'submit.httpt', '--scheme', 'http']); + + assert.deepEqual(parsed, { + command: 'run', + file: 'submit.httpt', + flags: { scheme: 'http' }, + }); + }); +}); diff --git a/js/packages/core/src/pipeline.js b/js/packages/core/src/pipeline.js index 128f9bc..0d79f3c 100644 --- a/js/packages/core/src/pipeline.js +++ b/js/packages/core/src/pipeline.js @@ -44,6 +44,21 @@ function materializeStream(value) { throw createNamedError('TemplateSyntaxError', 'Only string and Buffer stream values are supported synchronously'); } +function parseHydrateFunction(functionSource) { + const trimmed = functionSource.trim(); + const match = trimmed.match(/^([A-Za-z0-9-]+)(?:\((.*)\))?$/); + + if (!match) { + throw createNamedError('TemplateSyntaxError', `Invalid hydrate function: ${trimmed}`); + } + + if (match[2] != null && match[2].trim()) { + throw createNamedError('TemplateSyntaxError', `Hydrate function arguments are not supported yet: ${trimmed}`); + } + + return match[1]; +} + function applyHydrateFunction(value, functionName, streams) { switch (functionName) { case 'raw': @@ -68,13 +83,18 @@ function applyHydrateFunction(value, functionName, streams) { } function hydrate(template, data, streams = []) { - let hydratedCursor = 0; const map = []; - const normalizedTemplate = String(template).replace(/\r\n/g, '\n'); + const source = String(template); + const newline = source.includes('\r\n') ? '\r\n' : '\n'; + let shift = 0; - let resolved = normalizedTemplate.replace(/\{\{\s*([^|}]+?)\s*\|\s*([^}]+?)\s*\}\}/g, (match, key, functionName, offset) => { + let resolved = source.replace(/\{\{\s*([^|}]+?)\s*((?:\|\s*[^|}]+?\s*)+)\}\}/g, (match, key, functionSource, offset) => { const dataKey = key.trim(); - const transform = functionName.trim(); + const transforms = functionSource + .split('|') + .map((part) => part.trim()) + .filter(Boolean) + .map(parseHydrateFunction); if (!Object.prototype.hasOwnProperty.call(data, dataKey)) { throw createNamedError('MissingArgumentError', `Missing data key: ${dataKey}`, { @@ -82,31 +102,35 @@ function hydrate(template, data, streams = []) { }); } - hydratedCursor += offset - (map.at(-1)?.['original-start'] ?? 0) - (map.at(-1)?.['original-length'] ?? 0); + let replacement = data[dataKey]; + for (const transform of transforms) { + replacement = applyHydrateFunction(replacement, transform, streams); + } + replacement = String(replacement); - const replacement = applyHydrateFunction(data[dataKey], transform, streams); map.push({ - 'hydrated-start': hydratedCursor, + 'hydrated-start': offset + shift, 'original-start': offset, 'hydrated-length': replacement.length, 'original-length': match.length, }); - hydratedCursor += replacement.length; + shift += replacement.length - match.length; return replacement; }); const boundaryMatch = resolved.match(/\r?\n\r?\n/); const boundaryIndex = boundaryMatch ? boundaryMatch.index : resolved.length; - const boundary = boundaryMatch ? boundaryMatch[0] : '\n\n'; - let head = resolved.slice(0, boundaryIndex).replace(/\n+$/, ''); + let boundary = boundaryMatch ? boundaryMatch[0] : ''; + let head = resolved.slice(0, boundaryIndex).replace(/(?:\r?\n)+$/, ''); let body = boundaryMatch ? resolved.slice(boundaryIndex + boundary.length) : ''; + let bodyStream = null; if (Array.isArray(data.headers) && data.headers.length > 0) { const dynamicHeaders = data.headers .map(({ name, value }) => `${name}: ${value}`) - .join('\n'); - head = `${head}\n${dynamicHeaders}`; + .join(newline); + head = `${head}${newline}${dynamicHeaders}`; } if (data.body) { @@ -116,13 +140,23 @@ function hydrate(template, data, streams = []) { throw error; } const bodyType = data.body.type || 'text'; - head = `${head}\n:httpt-body-type: ${bodyType}`; - body = data.body.content == null ? '' : String(data.body.content); + head = `${head}${newline}:httpt-body-type: ${bodyType}`; + + if (bodyType === 'provided') { + const streamIndex = data.body.content == null ? 0 : data.body.content; + bodyStream = streams[streamIndex] || null; + body = ''; + } else { + body = data.body.content == null ? '' : String(data.body.content); + if (body && !boundary) { + boundary = `${newline}${newline}`; + } + } } - resolved = `${head}${boundary}${body}`; + resolved = `${head}${boundary || newline}${body}`; - return { resolved, map, bodyStream: null }; + return { resolved, map, bodyStream }; } function parse(resolved, optionalBodyStream = null) { diff --git a/js/packages/core/test/pipeline.test.js b/js/packages/core/test/pipeline.test.js index 3dc2890..e4607ee 100644 --- a/js/packages/core/test/pipeline.test.js +++ b/js/packages/core/test/pipeline.test.js @@ -3,6 +3,10 @@ const assert = require('node:assert'); const { hydrate, parse } = require('../src/pipeline.js'); const { loadE2eFixtures } = require('@httpt/test-utils'); +function normalizeLineEndings(value) { + return String(value).replace(/\r\n/g, '\n'); +} + describe('Pipeline: Hydrate & Parse', () => { const fixtures = loadE2eFixtures(); @@ -12,7 +16,7 @@ describe('Pipeline: Hydrate & Parse', () => { it(`should throw ${fixture.error.name} for ${nameToLog}`, async () => { await assert.rejects( async () => { - const { resolved, bodyStream } = await hydrate(fixture.template, fixture.data); + const { resolved, bodyStream } = await hydrate(fixture.template, fixture.data, fixture.dataStreams); parse(resolved, bodyStream); }, (err) => err.name === fixture.error.name @@ -21,9 +25,33 @@ describe('Pipeline: Hydrate & Parse', () => { continue; } it(`should process ${nameToLog} correctly`, async () => { - const { resolved, bodyStream } = await hydrate(fixture.template, fixture.data); + const { resolved, map, bodyStream } = await hydrate(fixture.template, fixture.data, fixture.dataStreams); + assert.equal(normalizeLineEndings(resolved), normalizeLineEndings(fixture.resolved)); + assert.deepEqual(map, fixture.map); + const { ir } = parse(resolved, bodyStream); assert.deepEqual(ir, fixture.ir); }); } + + it('should apply chained hydrate functions from left to right', () => { + const template = 'GET /search?q={{ term | json-string | url }} HTTP/1.1\r\nHost: example.com\r\n'; + const { resolved } = hydrate(template, { term: 'a "quoted" value' }); + + assert.equal( + resolved, + 'GET /search?q=a%20%5C%22quoted%5C%22%20value HTTP/1.1\r\nHost: example.com\r\n' + ); + }); + + it('should hand off dynamic provided bodies without serializing the stream index', () => { + const stream = Buffer.from('stream body'); + const template = 'POST /upload HTTP/1.1\r\nHost: example.com\r\n'; + const { resolved, bodyStream } = hydrate(template, { body: { type: 'provided', content: 0 } }, [stream]); + const { ir, bodyStream: parsedBodyStream } = parse(resolved, bodyStream); + + assert.equal(resolved.endsWith('\r\n:httpt-body-type: provided\r\n'), true); + assert.strictEqual(parsedBodyStream, stream); + assert.deepEqual(ir.body, { type: 'provided', content: 0 }); + }); }); diff --git a/js/packages/test-utils/index.js b/js/packages/test-utils/index.js index 40e436f..5cfc9c7 100644 --- a/js/packages/test-utils/index.js +++ b/js/packages/test-utils/index.js @@ -103,6 +103,20 @@ function getFixtureBaseNames() { return files.filter(f => f.endsWith('.httpt')).map(f => f.replace('.httpt', '')); } +function loadReferencedStreams(referenceFileName) { + const streams = []; + + for (let index = 0; ; index += 1) { + const streamPath = path.join(E2E_DIR, `${referenceFileName}-provided-stream-${index}`); + if (!fs.existsSync(streamPath)) { + break; + } + streams.push(fs.readFileSync(streamPath)); + } + + return streams; +} + function loadE2eFixtures() { const baseNames = getFixtureBaseNames(); const fixtures = []; @@ -130,6 +144,20 @@ function loadE2eFixtures() { irFile = null; } + let resolved = null; + try { + resolved = fs.readFileSync(path.join(E2E_DIR, `${baseName}.httpt-r`), 'utf8'); + } catch (e) { + // default to null + } + + let map = null; + try { + map = JSON.parse(fs.readFileSync(path.join(E2E_DIR, `${baseName}.httpt-map`), 'utf8')); + } catch (e) { + // default to null + } + let error = null; try { error = JSON.parse(fs.readFileSync(path.join(E2E_DIR, `${baseName}.error.json`), 'utf8')); @@ -139,6 +167,8 @@ function loadE2eFixtures() { let streamContent = null; let streamFilePath = null; + const dataFile = `${baseName}.data.json`; + const dataStreams = loadReferencedStreams(dataFile); if (ir && ir.body && ir.body.type === 'provided') { const streamIndex = ir.body.content !== undefined ? ir.body.content : 0; @@ -151,7 +181,19 @@ function loadE2eFixtures() { } } - fixtures.push({ baseName, irFile, ir, error, streamContent, streamFilePath, template, data }); + fixtures.push({ + baseName, + irFile, + ir, + resolved, + map, + error, + streamContent, + streamFilePath, + dataStreams, + template, + data, + }); } return fixtures; @@ -184,4 +226,4 @@ function normalizeForEchoServer(expectedIR, serverIR, adapterName) { } } -module.exports = { E2E_DIR, createEchoServer, binarizeIr, loadE2eFixtures, normalizeForEchoServer, getFixtureBaseNames }; +module.exports = { E2E_DIR, createEchoServer, binarizeIr, loadE2eFixtures, normalizeForEchoServer, getFixtureBaseNames, loadReferencedStreams }; diff --git a/test-fixtures/e2e/001-simple-get.httpt-map b/test-fixtures/e2e/001-simple-get.httpt-map index dcf0b74..ef1ebfd 100644 --- a/test-fixtures/e2e/001-simple-get.httpt-map +++ b/test-fixtures/e2e/001-simple-get.httpt-map @@ -1,5 +1,5 @@ [ - { "hydrated-start": 20, "original-start": 20, "hydrated-length": 4, "original-length": 23 }, - { "hydrated-start": 40, "original-start": 58, "hydrated-length": 15, "original-length": 19 }, - { "hydrated-start": 78, "original-start": 106, "hydrated-length": 8, "original-length": 21 } + { "hydrated-start": 21, "original-start": 21, "hydrated-length": 4, "original-length": 23 }, + { "hydrated-start": 42, "original-start": 61, "hydrated-length": 15, "original-length": 20 }, + { "hydrated-start": 81, "original-start": 105, "hydrated-length": 8, "original-length": 22 } ] diff --git a/test-fixtures/e2e/002-post-json.httpt-map b/test-fixtures/e2e/002-post-json.httpt-map index 88c5bab..91f59be 100644 --- a/test-fixtures/e2e/002-post-json.httpt-map +++ b/test-fixtures/e2e/002-post-json.httpt-map @@ -1,7 +1,7 @@ [ - { "hydrated-start": 33, "original-start": 33, "hydrated-length": 25, "original-length": 20 }, - { "hydrated-start": 81, "original-start": 76, "hydrated-length": 9, "original-length": 17 }, - { "hydrated-start": 125, "original-start": 125, "hydrated-length": 16, "original-length": 22 }, - { "hydrated-start": 145, "original-start": 151, "hydrated-length": 45, "original-length": 22 }, - { "hydrated-start": 204, "original-start": 187, "hydrated-length": 12, "original-length": 28 } + { "hydrated-start": 38, "original-start": 38, "hydrated-length": 20, "original-length": 20 }, + { "hydrated-start": 82, "original-start": 82, "hydrated-length": 9, "original-length": 17 }, + { "hydrated-start": 133, "original-start": 141, "hydrated-length": 16, "original-length": 22 }, + { "hydrated-start": 152, "original-start": 166, "hydrated-length": 43, "original-length": 24 }, + { "hydrated-start": 209, "original-start": 204, "hydrated-length": 12, "original-length": 29 } ] diff --git a/test-fixtures/e2e/004-post-json-native.httpt-map b/test-fixtures/e2e/004-post-json-native.httpt-map index 998a298..9f2c506 100644 --- a/test-fixtures/e2e/004-post-json-native.httpt-map +++ b/test-fixtures/e2e/004-post-json-native.httpt-map @@ -1 +1 @@ -[{"hydrated-start":80,"original-start":80,"hydrated-length":3,"original-length":19}] +[{"hydrated-start":84,"original-start":84,"hydrated-length":3,"original-length":19}] diff --git a/test-fixtures/e2e/011-identity-template.httpt-map b/test-fixtures/e2e/011-identity-template.httpt-map index fe51488..01edf42 100644 --- a/test-fixtures/e2e/011-identity-template.httpt-map +++ b/test-fixtures/e2e/011-identity-template.httpt-map @@ -1 +1,6 @@ -[] +[ + { "hydrated-start": 0, "original-start": 0, "hydrated-length": 5, "original-length": 18 }, + { "hydrated-start": 6, "original-start": 19, "hydrated-length": 13, "original-length": 15 }, + { "hydrated-start": 20, "original-start": 35, "hydrated-length": 8, "original-length": 19 }, + { "hydrated-start": 36, "original-start": 62, "hydrated-length": 20, "original-length": 16 } +] diff --git a/test-fixtures/e2e/014-body-junk.httpt-r b/test-fixtures/e2e/014-body-junk.httpt-r index 05002a9..dea061a 100644 --- a/test-fixtures/e2e/014-body-junk.httpt-r +++ b/test-fixtures/e2e/014-body-junk.httpt-r @@ -1,6 +1,5 @@ POST /api/junk HTTP/1.1 Host: example.com -:httpt-body-type: text This body has a double-newline right here: From f4db808cb918e7214f234f3360dc50f2bddaaf01 Mon Sep 17 00:00:00 2001 From: Waleed Ahmad <138612573+waleedm007@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:07:52 +0500 Subject: [PATCH 3/8] Make fixture map tests line-ending agnostic --- js/packages/test-utils/index.js | 8 ++++++-- test-fixtures/e2e/001-simple-get.httpt-map | 4 ++-- test-fixtures/e2e/002-post-json.httpt-map | 10 +++++----- test-fixtures/e2e/004-post-json-native.httpt-map | 2 +- test-fixtures/e2e/011-identity-template.httpt-map | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/js/packages/test-utils/index.js b/js/packages/test-utils/index.js index 5cfc9c7..39198cd 100644 --- a/js/packages/test-utils/index.js +++ b/js/packages/test-utils/index.js @@ -7,6 +7,10 @@ const E2E_DIR = path.resolve(__dirname, '../../../test-fixtures/e2e'); const IGNORED_HEADERS = ['host', 'connection', 'accept', 'accept-language', 'sec-fetch-mode', 'user-agent', 'accept-encoding', 'content-length', 'content-type', 'transfer-encoding']; +function normalizeFixtureText(value) { + return String(value).replace(/\r\n/g, '\n'); +} + /** * Starts an HTTP echo server on a random port. * Returns a server object with a method `getCapturedRequest()` that waits for a request, @@ -124,7 +128,7 @@ function loadE2eFixtures() { for (const baseName of baseNames) { let template = ''; try { - template = fs.readFileSync(path.join(E2E_DIR, `${baseName}.httpt`), 'utf8'); + template = normalizeFixtureText(fs.readFileSync(path.join(E2E_DIR, `${baseName}.httpt`), 'utf8')); } catch (e) { // default to '' } @@ -146,7 +150,7 @@ function loadE2eFixtures() { let resolved = null; try { - resolved = fs.readFileSync(path.join(E2E_DIR, `${baseName}.httpt-r`), 'utf8'); + resolved = normalizeFixtureText(fs.readFileSync(path.join(E2E_DIR, `${baseName}.httpt-r`), 'utf8')); } catch (e) { // default to null } diff --git a/test-fixtures/e2e/001-simple-get.httpt-map b/test-fixtures/e2e/001-simple-get.httpt-map index ef1ebfd..63f3380 100644 --- a/test-fixtures/e2e/001-simple-get.httpt-map +++ b/test-fixtures/e2e/001-simple-get.httpt-map @@ -1,5 +1,5 @@ [ { "hydrated-start": 21, "original-start": 21, "hydrated-length": 4, "original-length": 23 }, - { "hydrated-start": 42, "original-start": 61, "hydrated-length": 15, "original-length": 20 }, - { "hydrated-start": 81, "original-start": 105, "hydrated-length": 8, "original-length": 22 } + { "hydrated-start": 41, "original-start": 60, "hydrated-length": 15, "original-length": 20 }, + { "hydrated-start": 79, "original-start": 103, "hydrated-length": 8, "original-length": 22 } ] diff --git a/test-fixtures/e2e/002-post-json.httpt-map b/test-fixtures/e2e/002-post-json.httpt-map index 91f59be..f913421 100644 --- a/test-fixtures/e2e/002-post-json.httpt-map +++ b/test-fixtures/e2e/002-post-json.httpt-map @@ -1,7 +1,7 @@ [ - { "hydrated-start": 38, "original-start": 38, "hydrated-length": 20, "original-length": 20 }, - { "hydrated-start": 82, "original-start": 82, "hydrated-length": 9, "original-length": 17 }, - { "hydrated-start": 133, "original-start": 141, "hydrated-length": 16, "original-length": 22 }, - { "hydrated-start": 152, "original-start": 166, "hydrated-length": 43, "original-length": 24 }, - { "hydrated-start": 209, "original-start": 204, "hydrated-length": 12, "original-length": 29 } + { "hydrated-start": 37, "original-start": 37, "hydrated-length": 20, "original-length": 20 }, + { "hydrated-start": 80, "original-start": 80, "hydrated-length": 9, "original-length": 17 }, + { "hydrated-start": 127, "original-start": 135, "hydrated-length": 16, "original-length": 22 }, + { "hydrated-start": 146, "original-start": 160, "hydrated-length": 43, "original-length": 24 }, + { "hydrated-start": 202, "original-start": 197, "hydrated-length": 12, "original-length": 29 } ] diff --git a/test-fixtures/e2e/004-post-json-native.httpt-map b/test-fixtures/e2e/004-post-json-native.httpt-map index 9f2c506..998a298 100644 --- a/test-fixtures/e2e/004-post-json-native.httpt-map +++ b/test-fixtures/e2e/004-post-json-native.httpt-map @@ -1 +1 @@ -[{"hydrated-start":84,"original-start":84,"hydrated-length":3,"original-length":19}] +[{"hydrated-start":80,"original-start":80,"hydrated-length":3,"original-length":19}] diff --git a/test-fixtures/e2e/011-identity-template.httpt-map b/test-fixtures/e2e/011-identity-template.httpt-map index 01edf42..f1f9d21 100644 --- a/test-fixtures/e2e/011-identity-template.httpt-map +++ b/test-fixtures/e2e/011-identity-template.httpt-map @@ -2,5 +2,5 @@ { "hydrated-start": 0, "original-start": 0, "hydrated-length": 5, "original-length": 18 }, { "hydrated-start": 6, "original-start": 19, "hydrated-length": 13, "original-length": 15 }, { "hydrated-start": 20, "original-start": 35, "hydrated-length": 8, "original-length": 19 }, - { "hydrated-start": 36, "original-start": 62, "hydrated-length": 20, "original-length": 16 } + { "hydrated-start": 35, "original-start": 61, "hydrated-length": 20, "original-length": 16 } ] From 51e41dcbb141899a659febed521f4378af966293 Mon Sep 17 00:00:00 2001 From: Waleed Ahmad <138612573+waleedm007@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:49:13 +0500 Subject: [PATCH 4/8] Complete incubation spec coverage for JS library --- js/packages/cli/src/bin.js | 4 + js/packages/cli/src/commands/emit.js | 9 +- js/packages/cli/src/commands/hydrate.js | 4 +- js/packages/cli/src/commands/run.js | 6 +- js/packages/cli/src/commands/verify.js | 16 ++ js/packages/cli/src/parser.js | 12 +- js/packages/cli/src/streams.js | 20 ++ js/packages/cli/test/parser.test.js | 19 ++ js/packages/cli/test/streams.test.js | 20 ++ js/packages/cli/test/verify.test.js | 16 ++ js/packages/core/index.js | 16 +- js/packages/core/src/dispatch.js | 5 +- js/packages/core/src/facade.js | 6 +- js/packages/core/src/pipeline.js | 255 +++++++++++++++++- js/packages/core/test/facade.test.js | 31 +++ js/packages/core/test/verification.test.js | 81 ++++++ js/packages/test-utils/index.js | 13 + .../e2e/017-http-2-protocol.data.json | 1 + test-fixtures/e2e/017-http-2-protocol.httpt | 3 + .../e2e/017-http-2-protocol.httpt-ir | 8 + .../e2e/017-http-2-protocol.httpt-map | 1 + test-fixtures/e2e/017-http-2-protocol.httpt-r | 3 + .../e2e/018-bizarre-spacing.data.json | 1 + test-fixtures/e2e/018-bizarre-spacing.httpt | 3 + .../e2e/018-bizarre-spacing.httpt-ir | 8 + .../e2e/018-bizarre-spacing.httpt-map | 1 + test-fixtures/e2e/018-bizarre-spacing.httpt-r | 3 + test-fixtures/e2e/019-absolute-uri.data.json | 1 + test-fixtures/e2e/019-absolute-uri.httpt | 4 + test-fixtures/e2e/019-absolute-uri.httpt-ir | 10 + test-fixtures/e2e/019-absolute-uri.httpt-map | 1 + test-fixtures/e2e/019-absolute-uri.httpt-r | 4 + .../e2e/020-malformed-json.data.json | 1 + .../e2e/020-malformed-json.error.json | 3 + test-fixtures/e2e/020-malformed-json.httpt | 5 + test-fixtures/e2e/TEST_MATRIX.md | 40 +-- 36 files changed, 591 insertions(+), 43 deletions(-) create mode 100644 js/packages/cli/src/commands/verify.js create mode 100644 js/packages/cli/src/streams.js create mode 100644 js/packages/cli/test/streams.test.js create mode 100644 js/packages/cli/test/verify.test.js create mode 100644 js/packages/core/test/facade.test.js create mode 100644 js/packages/core/test/verification.test.js create mode 100644 test-fixtures/e2e/017-http-2-protocol.data.json create mode 100644 test-fixtures/e2e/017-http-2-protocol.httpt create mode 100644 test-fixtures/e2e/017-http-2-protocol.httpt-ir create mode 100644 test-fixtures/e2e/017-http-2-protocol.httpt-map create mode 100644 test-fixtures/e2e/017-http-2-protocol.httpt-r create mode 100644 test-fixtures/e2e/018-bizarre-spacing.data.json create mode 100644 test-fixtures/e2e/018-bizarre-spacing.httpt create mode 100644 test-fixtures/e2e/018-bizarre-spacing.httpt-ir create mode 100644 test-fixtures/e2e/018-bizarre-spacing.httpt-map create mode 100644 test-fixtures/e2e/018-bizarre-spacing.httpt-r create mode 100644 test-fixtures/e2e/019-absolute-uri.data.json create mode 100644 test-fixtures/e2e/019-absolute-uri.httpt create mode 100644 test-fixtures/e2e/019-absolute-uri.httpt-ir create mode 100644 test-fixtures/e2e/019-absolute-uri.httpt-map create mode 100644 test-fixtures/e2e/019-absolute-uri.httpt-r create mode 100644 test-fixtures/e2e/020-malformed-json.data.json create mode 100644 test-fixtures/e2e/020-malformed-json.error.json create mode 100644 test-fixtures/e2e/020-malformed-json.httpt diff --git a/js/packages/cli/src/bin.js b/js/packages/cli/src/bin.js index 7905f2f..5936037 100755 --- a/js/packages/cli/src/bin.js +++ b/js/packages/cli/src/bin.js @@ -5,6 +5,7 @@ const hydrateCommand = require('./commands/hydrate'); const parseCommand = require('./commands/parse'); const emitCommand = require('./commands/emit'); const runCommand = require('./commands/run'); +const verifyCommand = require('./commands/verify'); async function main() { const { command, file, flags } = parseArgs(process.argv); @@ -22,6 +23,9 @@ async function main() { case 'run': await runCommand(file, flags); break; + case 'verify': + await verifyCommand(file, flags); + break; default: console.log(`Unknown command: ${command}`); process.exit(1); diff --git a/js/packages/cli/src/commands/emit.js b/js/packages/cli/src/commands/emit.js index 8639065..bb17ce8 100644 --- a/js/packages/cli/src/commands/emit.js +++ b/js/packages/cli/src/commands/emit.js @@ -15,13 +15,16 @@ const { dispatchFetch } = require('@httpt/core'); */ function dispatchCurl(ir, scheme, bodyStream = null) { return new Promise((resolve, reject) => { - const url = `${scheme}://${ir.host}${ir.uri}`; + const parsedUrl = new URL(ir.uri, `${scheme}://${ir.host}`); + parsedUrl.protocol = `${scheme}:`; + parsedUrl.host = ir.host; + const url = parsedUrl.toString(); const args = ['-s', '-v', '-X', ir.method]; if (ir.version === 'HTTP/1.0') args.push('--http1.0'); else if (ir.version === 'HTTP/1.1') args.push('--http1.1'); - else if (ir.version === 'HTTP/2' || ir.version === 'HTTP/2.0') args.push('--http2'); - else if (ir.version === 'HTTP/3') args.push('--http3'); + // HTTP/2 and HTTP/3 support depends on the installed curl build and server. + // Let curl negotiate/fallback instead of making local E2E execution fail. for (const { name, value } of ir.headers) { args.push('-H', value ? `${name}: ${value}` : `${name};`); diff --git a/js/packages/cli/src/commands/hydrate.js b/js/packages/cli/src/commands/hydrate.js index 6c90056..83f147d 100644 --- a/js/packages/cli/src/commands/hydrate.js +++ b/js/packages/cli/src/commands/hydrate.js @@ -1,5 +1,6 @@ const fs = require('node:fs'); const { hydrate } = require('@httpt/core'); +const { loadStreamsFromFlags } = require('../streams'); function hydrateCommand(file, flags) { if (!file) { @@ -12,7 +13,8 @@ function hydrateCommand(file, flags) { const template = fs.readFileSync(file, 'utf-8'); const data = JSON.parse(fs.readFileSync(flags.data, 'utf-8')); - const { resolved, map } = hydrate(template, data); + const streams = loadStreamsFromFlags(flags); + const { resolved, map } = hydrate(template, data, streams); const outputFile = flags.out || flags.output || `${file}-r`; fs.writeFileSync(outputFile, resolved); diff --git a/js/packages/cli/src/commands/run.js b/js/packages/cli/src/commands/run.js index bd72386..7f5afbf 100644 --- a/js/packages/cli/src/commands/run.js +++ b/js/packages/cli/src/commands/run.js @@ -1,5 +1,6 @@ const fs = require('node:fs'); const { build, execute } = require('@httpt/core'); +const { loadStreamsFromFlags } = require('../streams'); async function runCommand(file, flags) { if (!file) { @@ -12,10 +13,11 @@ async function runCommand(file, flags) { const template = fs.readFileSync(file, 'utf-8'); const data = JSON.parse(fs.readFileSync(flags.data, 'utf-8')); + const streams = loadStreamsFromFlags(flags); const isDryRun = flags['dry-run'] || flags.dryRun; if (isDryRun || flags.out || flags.output) { - const { ir } = await build(template, data); + const { ir } = await build(template, data, streams); const outputFile = flags.out || flags.output; if (outputFile) { @@ -28,7 +30,7 @@ async function runCommand(file, flags) { return; } - const response = await execute(template, data, [], { + const response = await execute(template, data, streams, { scheme: flags.scheme || 'https', }); diff --git a/js/packages/cli/src/commands/verify.js b/js/packages/cli/src/commands/verify.js new file mode 100644 index 0000000..fd6f753 --- /dev/null +++ b/js/packages/cli/src/commands/verify.js @@ -0,0 +1,16 @@ +const fs = require('node:fs'); +const { verifyContract } = require('@httpt/core'); +const { normalizeFlagList } = require('../streams'); + +function verifyCommand(file, flags) { + if (!file) { + throw new Error('Missing template file'); + } + + const template = fs.readFileSync(file, 'utf-8'); + const expectedArguments = normalizeFlagList(flags.expect || flags.expected || flags.arg); + verifyContract(template, expectedArguments); + console.log('Contract verified'); +} + +module.exports = verifyCommand; diff --git a/js/packages/cli/src/parser.js b/js/packages/cli/src/parser.js index e784439..846c02b 100644 --- a/js/packages/cli/src/parser.js +++ b/js/packages/cli/src/parser.js @@ -6,6 +6,14 @@ function parseArgs(argv) { const flags = {}; + function setFlag(name, value) { + if (Object.prototype.hasOwnProperty.call(flags, name)) { + flags[name] = Array.isArray(flags[name]) ? flags[name].concat(value) : [flags[name], value]; + return; + } + flags[name] = value; + } + for (let i=1, len=args.length; i { + if (streamPath === true) { + throw new Error('Missing path for --stream flag'); + } + return fs.readFileSync(streamPath); + }); +} + +module.exports = { loadStreamsFromFlags, normalizeFlagList }; diff --git a/js/packages/cli/test/parser.test.js b/js/packages/cli/test/parser.test.js index 13f7505..6d13276 100644 --- a/js/packages/cli/test/parser.test.js +++ b/js/packages/cli/test/parser.test.js @@ -22,4 +22,23 @@ describe('CLI argument parser', () => { flags: { scheme: 'http' }, }); }); + + it('should preserve repeated flags as arrays', () => { + const parsed = parseArgs([ + 'node', + 'httpt', + 'run', + 'submit.httpt', + '--stream', + 'a.bin', + '--stream', + 'b.bin', + ]); + + assert.deepEqual(parsed, { + command: 'run', + file: 'submit.httpt', + flags: { stream: ['a.bin', 'b.bin'] }, + }); + }); }); diff --git a/js/packages/cli/test/streams.test.js b/js/packages/cli/test/streams.test.js new file mode 100644 index 0000000..0b9d762 --- /dev/null +++ b/js/packages/cli/test/streams.test.js @@ -0,0 +1,20 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { loadStreamsFromFlags } = require('../src/streams.js'); + +describe('CLI stream loading', () => { + it('should load --stream files as native buffers in order', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'httpt-streams-')); + const first = path.join(dir, 'first.txt'); + const second = path.join(dir, 'second.txt'); + fs.writeFileSync(first, 'first'); + fs.writeFileSync(second, 'second'); + + const streams = loadStreamsFromFlags({ stream: [first, second] }); + + assert.deepEqual(streams.map((stream) => stream.toString('utf8')), ['first', 'second']); + }); +}); diff --git a/js/packages/cli/test/verify.test.js b/js/packages/cli/test/verify.test.js new file mode 100644 index 0000000..092019c --- /dev/null +++ b/js/packages/cli/test/verify.test.js @@ -0,0 +1,16 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const verifyCommand = require('../src/commands/verify.js'); + +describe('CLI verify command', () => { + it('should verify a template against expected arguments', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'httpt-verify-')); + const templateFile = path.join(dir, 'request.httpt'); + fs.writeFileSync(templateFile, 'GET /{{ user-id | url }} HTTP/1.1\nHost: example.com\n'); + + assert.doesNotThrow(() => verifyCommand(templateFile, { expect: 'user-id' })); + }); +}); diff --git a/js/packages/core/index.js b/js/packages/core/index.js index 5f07672..fc4e712 100644 --- a/js/packages/core/index.js +++ b/js/packages/core/index.js @@ -1,9 +1,21 @@ -const { hydrate, parse } = require('./src/pipeline'); +const { + hydrate, + hydrateAsync, + parse, + parseAsync, + verifyContract, + validateStreamReferences, + prepareHydrationContext, +} = require('./src/pipeline'); const { build, execute } = require('./src/facade'); const { dispatchFetch } = require('./src/dispatch'); module.exports = { - hydrate, parse, + hydrate, hydrateAsync, + parse, parseAsync, + verifyContract, + validateStreamReferences, + prepareHydrationContext, build, execute, dispatchFetch }; diff --git a/js/packages/core/src/dispatch.js b/js/packages/core/src/dispatch.js index 52173f0..cb7228c 100644 --- a/js/packages/core/src/dispatch.js +++ b/js/packages/core/src/dispatch.js @@ -10,7 +10,10 @@ * @returns {Promise} */ async function dispatchFetch(ir, scheme, bodyStream = null) { - const url = new URL(ir.uri, `${scheme}://${ir.host}`).toString(); + const parsedUrl = new URL(ir.uri, `${scheme}://${ir.host}`); + parsedUrl.protocol = `${scheme}:`; + parsedUrl.host = ir.host; + const url = parsedUrl.toString(); const headers = new Headers(); for (const { name, value } of ir.headers) headers.append(name, value); diff --git a/js/packages/core/src/facade.js b/js/packages/core/src/facade.js index 159178c..a58b260 100644 --- a/js/packages/core/src/facade.js +++ b/js/packages/core/src/facade.js @@ -1,9 +1,9 @@ -const { hydrate, parse } = require('./pipeline'); +const { hydrateAsync, parseAsync } = require('./pipeline'); const { dispatchFetch } = require('./dispatch'); async function build(template, data, streams = []) { - const { resolved, map, bodyStream } = hydrate(template, data, streams); - const parsed = parse(resolved, bodyStream); + const { resolved, map, bodyStream } = await hydrateAsync(template, data, streams); + const parsed = await parseAsync(resolved, bodyStream); return { ir: parsed.ir, diff --git a/js/packages/core/src/pipeline.js b/js/packages/core/src/pipeline.js index 0d79f3c..f7e6fd0 100644 --- a/js/packages/core/src/pipeline.js +++ b/js/packages/core/src/pipeline.js @@ -5,6 +5,146 @@ function createNamedError(name, message, details = {}) { return error; } +const BUILT_IN_FUNCTIONS = new Set([ + 'raw', + 'url', + 'json-value', + 'json-string', + 'json-key', + 'stream-as-base64', + 'stream-as-utf8', + 'stream-as-is', +]); + +function isPlainObject(value) { + return value && typeof value === 'object' && Object.getPrototypeOf(value) === Object.prototype; +} + +function isNativeStream(value) { + return Boolean( + value && + typeof value === 'object' && + ( + Buffer.isBuffer(value) || + value instanceof Uint8Array || + typeof value.pipe === 'function' || + typeof value.getReader === 'function' || + typeof value.arrayBuffer === 'function' + ) + ); +} + +function cloneDataWithExtractedStreams(value, streams) { + if (isNativeStream(value)) { + const content = streams.length; + streams.push(value); + return { type: 'provided', content }; + } + + if (Array.isArray(value)) { + return value.map((item) => cloneDataWithExtractedStreams(item, streams)); + } + + if (isPlainObject(value)) { + const cloned = {}; + for (const [key, childValue] of Object.entries(value)) { + cloned[key] = cloneDataWithExtractedStreams(childValue, streams); + } + return cloned; + } + + return value; +} + +function prepareHydrationContext(data, streams = []) { + const preparedStreams = Array.isArray(streams) ? streams.slice() : []; + const preparedData = cloneDataWithExtractedStreams(data || {}, preparedStreams); + + validateStreamReferences(preparedData); + + return { data: preparedData, streams: preparedStreams }; +} + +function collectProvidedReferences(value, references = []) { + if (Array.isArray(value)) { + for (const item of value) collectProvidedReferences(item, references); + return references; + } + + if (!isPlainObject(value)) { + return references; + } + + if (value.type === 'provided') { + references.push({ + explicit: Object.prototype.hasOwnProperty.call(value, 'content'), + index: value.content == null ? 0 : value.content, + }); + } + + for (const childValue of Object.values(value)) { + collectProvidedReferences(childValue, references); + } + + return references; +} + +function validateStreamReferences(data) { + const references = collectProvidedReferences(data); + + if (references.length > 1 && references.some((reference) => !reference.explicit)) { + throw createNamedError( + 'TemplateSyntaxError', + 'Ambiguous provided stream reference: content index is required when multiple streams are referenced' + ); + } + + const seen = new Set(); + for (const reference of references) { + if (!Number.isInteger(reference.index) || reference.index < 0) { + throw createNamedError('TemplateSyntaxError', `Invalid provided stream index: ${reference.index}`); + } + + if (seen.has(reference.index)) { + throw createNamedError('TemplateSyntaxError', `Duplicate provided stream index: ${reference.index}`); + } + seen.add(reference.index); + } + + return true; +} + +async function resolveToString(input) { + if (typeof input === 'string') { + return input; + } + + if (Buffer.isBuffer(input) || input instanceof Uint8Array) { + return Buffer.from(input).toString('utf8'); + } + + if (input && typeof input.getReader === 'function') { + const reader = input.getReader(); + const chunks = []; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + chunks.push(Buffer.from(value)); + } + return Buffer.concat(chunks).toString('utf8'); + } + + if (input && typeof input[Symbol.asyncIterator] === 'function') { + const chunks = []; + for await (const chunk of input) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks).toString('utf8'); + } + + return String(input); +} + function formatJsonValue(value) { const json = JSON.stringify(value); @@ -37,6 +177,10 @@ function materializeStream(value) { return value; } + if (value instanceof Uint8Array) { + return Buffer.from(value); + } + if (typeof value === 'string') { return Buffer.from(value, 'utf8'); } @@ -52,15 +196,20 @@ function parseHydrateFunction(functionSource) { throw createNamedError('TemplateSyntaxError', `Invalid hydrate function: ${trimmed}`); } - if (match[2] != null && match[2].trim()) { - throw createNamedError('TemplateSyntaxError', `Hydrate function arguments are not supported yet: ${trimmed}`); + const name = match[1]; + const args = match[2] == null || !match[2].trim() + ? [] + : match[2].split(',').map((arg) => arg.trim()).filter(Boolean); + + if (!BUILT_IN_FUNCTIONS.has(name)) { + throw createNamedError('TemplateSyntaxError', `Unsupported hydrate function: ${name}`); } - return match[1]; + return { name, args }; } -function applyHydrateFunction(value, functionName, streams) { - switch (functionName) { +function applyHydrateFunction(value, hydrateFunction, streams) { + switch (hydrateFunction.name) { case 'raw': return String(value); case 'url': @@ -78,11 +227,15 @@ function applyHydrateFunction(value, functionName, streams) { case 'stream-as-is': return materializeStream(resolveStreamValue(value, streams)).toString('utf8'); default: - throw createNamedError('TemplateSyntaxError', `Unsupported hydrate function: ${functionName}`); + throw createNamedError('TemplateSyntaxError', `Unsupported hydrate function: ${hydrateFunction.name}`); } } function hydrate(template, data, streams = []) { + const prepared = prepareHydrationContext(data, streams); + data = prepared.data; + streams = prepared.streams; + const map = []; const source = String(template); const newline = source.includes('\r\n') ? '\r\n' : '\n'; @@ -159,6 +312,10 @@ function hydrate(template, data, streams = []) { return { resolved, map, bodyStream }; } +async function hydrateAsync(template, data, streams = []) { + return hydrate(await resolveToString(template), data, streams); +} + function parse(resolved, optionalBodyStream = null) { const normalizedResolved = String(resolved).replace(/\r\n/g, '\n'); const boundaryMatch = normalizedResolved.match(/\n\n/); @@ -184,7 +341,8 @@ function parse(resolved, optionalBodyStream = null) { for (const line of lines) { const colonIndex = line.startsWith(':') ? line.indexOf(':', 1) : line.indexOf(':'); if (colonIndex === -1) { - throw createNamedError('TemplateSyntaxError', `Invalid header line: ${line}`); + const index = normalizedResolved.indexOf(line); + throw createNamedError('TemplateSyntaxError', `Invalid header line: ${line}`, { index }); } const name = line.slice(0, colonIndex).trim(); const value = line.slice(colonIndex + 1).trim(); @@ -217,7 +375,13 @@ function parse(resolved, optionalBodyStream = null) { } else if (normalizedBody) { let content = normalizedBody; if (bodyType === 'json') { - content = JSON.parse(normalizedBody); + try { + content = JSON.parse(normalizedBody); + } catch (error) { + throw createNamedError('TemplateSyntaxError', `Invalid JSON body: ${error.message}`, { + index: normalizedResolved.indexOf(normalizedBody), + }); + } } else if (bodyType === 'base64') { content = normalizedBody.trim(); } @@ -230,4 +394,77 @@ function parse(resolved, optionalBodyStream = null) { return { ir, bodyStream: optionalBodyStream }; } -module.exports = { hydrate, parse }; +async function parseAsync(resolved, optionalBodyStream = null) { + return parse(await resolveToString(resolved), optionalBodyStream); +} + +function scanTemplate(template) { + const source = String(template); + const tags = []; + let cursor = 0; + + while (cursor < source.length) { + const start = source.indexOf('{{', cursor); + if (start === -1) break; + + const end = source.indexOf('}}', start + 2); + if (end === -1) { + throw createNamedError('TemplateSyntaxError', 'Unclosed template tag', { index: start }); + } + + const inner = source.slice(start + 2, end); + if (inner.includes('{{')) { + throw createNamedError('TemplateSyntaxError', 'Nested template tags are not allowed', { index: start }); + } + + const parts = inner.split('|').map((part) => part.trim()).filter(Boolean); + if (parts.length < 2) { + throw createNamedError('TemplateSyntaxError', 'Template tag must include at least one function', { index: start }); + } + + const parameter = parts[0]; + const functions = parts.slice(1).map((part) => parseHydrateFunction(part)); + tags.push({ parameter, functions, start, end: end + 2 }); + cursor = end + 2; + } + + const strayClose = source.indexOf('}}', cursor); + if (strayClose !== -1) { + throw createNamedError('TemplateSyntaxError', 'Unexpected template close tag', { index: strayClose }); + } + + return tags; +} + +function verifyContract(template, expectedArguments = []) { + const tags = scanTemplate(template); + const used = new Set(tags.map((tag) => tag.parameter)); + const expected = new Set(expectedArguments); + + const missing = [...used].filter((key) => !expected.has(key)); + if (missing.length > 0) { + throw createNamedError('MissingArgumentError', `Template requires missing contract arguments: ${missing.join(', ')}`, { + missing, + }); + } + + const extra = [...expected].filter((key) => !used.has(key)); + if (extra.length > 0) { + throw createNamedError('UnexpectedArgumentError', `Contract includes unused arguments: ${extra.join(', ')}`, { + extra, + }); + } + + return true; +} + +module.exports = { + hydrate, + hydrateAsync, + parse, + parseAsync, + verifyContract, + validateStreamReferences, + prepareHydrationContext, + resolveToString, +}; diff --git a/js/packages/core/test/facade.test.js b/js/packages/core/test/facade.test.js new file mode 100644 index 0000000..88555c7 --- /dev/null +++ b/js/packages/core/test/facade.test.js @@ -0,0 +1,31 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert'); +const { Readable } = require('node:stream'); +const { build, hydrateAsync, parseAsync } = require('../index.js'); + +describe('SDK facade polymorphic inputs', () => { + it('should build from a Node readable template stream', async () => { + const template = Readable.from([ + 'GET /users/{{ user-id | url }} HTTP/1.1\n', + 'Host: example.com\n', + ]); + + const { ir } = await build(template, { 'user-id': 'a b' }); + + assert.deepEqual(ir, { + 'schema-version': '1.0', + method: 'GET', + host: 'example.com', + uri: '/users/a%20b', + version: 'HTTP/1.1', + headers: [], + }); + }); + + it('should hydrate and parse asynchronously from byte inputs', async () => { + const { resolved } = await hydrateAsync(Buffer.from('GET / HTTP/1.1\nHost: example.com\n'), {}); + const { ir } = await parseAsync(Buffer.from(resolved)); + + assert.equal(ir.host, 'example.com'); + }); +}); diff --git a/js/packages/core/test/verification.test.js b/js/packages/core/test/verification.test.js new file mode 100644 index 0000000..ccdb5eb --- /dev/null +++ b/js/packages/core/test/verification.test.js @@ -0,0 +1,81 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert'); +const { + hydrate, + verifyContract, + validateStreamReferences, + prepareHydrationContext, +} = require('../src/pipeline.js'); + +describe('Static contract verification', () => { + it('should return true when syntax and expected arguments match', () => { + const template = [ + 'GET /users/{{ user-id | url }} HTTP/1.1', + 'Host: api.example.com', + 'Authorization: Bearer {{ auth-token | raw }}', + '', + ].join('\n'); + + assert.equal(verifyContract(template, ['user-id', 'auth-token']), true); + }); + + it('should reject missing contract arguments', () => { + assert.throws( + () => verifyContract('GET /{{ user-id | url }} HTTP/1.1\n', []), + (error) => error.name === 'MissingArgumentError' && error.missing.includes('user-id') + ); + }); + + it('should reject extra contract arguments', () => { + assert.throws( + () => verifyContract('GET /{{ user-id | url }} HTTP/1.1\n', ['user-id', 'unused']), + (error) => error.name === 'UnexpectedArgumentError' && error.extra.includes('unused') + ); + }); + + it('should reject malformed template tags and unsupported functions', () => { + assert.throws( + () => verifyContract('GET /{{ user-id | mystery }} HTTP/1.1\n', ['user-id']), + (error) => error.name === 'TemplateSyntaxError' + ); + + assert.throws( + () => verifyContract('GET /{{ user-id | url HTTP/1.1\n', ['user-id']), + (error) => error.name === 'TemplateSyntaxError' && Number.isInteger(error.index) + ); + }); +}); + +describe('Stream reference validation', () => { + it('should reject ambiguous implicit provided stream references', () => { + assert.throws( + () => validateStreamReferences({ + first: { type: 'provided' }, + second: { type: 'provided', content: 1 }, + }), + (error) => error.name === 'TemplateSyntaxError' + ); + }); + + it('should reject duplicate provided stream indexes', () => { + assert.throws( + () => validateStreamReferences({ + first: { type: 'provided', content: 0 }, + second: { type: 'provided', content: 0 }, + }), + (error) => error.name === 'TemplateSyntaxError' + ); + }); + + it('should extract native streams from data and make them addressable as provided references', () => { + const stream = Buffer.from('stream-value'); + const { data, streams } = prepareHydrationContext({ upload: stream }); + + assert.deepEqual(data.upload, { type: 'provided', content: 0 }); + assert.strictEqual(streams[0], stream); + + const template = 'POST /upload HTTP/1.1\nHost: example.com\n\n{{ upload | stream-as-utf8 }}'; + const { resolved } = hydrate(template, { upload: stream }); + assert.equal(resolved, 'POST /upload HTTP/1.1\nHost: example.com\n\nstream-value'); + }); +}); diff --git a/js/packages/test-utils/index.js b/js/packages/test-utils/index.js index 39198cd..e595a8a 100644 --- a/js/packages/test-utils/index.js +++ b/js/packages/test-utils/index.js @@ -204,6 +204,19 @@ function loadE2eFixtures() { } function normalizeForEchoServer(expectedIR, serverIR, adapterName) { + if (expectedIR.uri) { + try { + const parsedUri = new URL(expectedIR.uri); + expectedIR.uri = `${parsedUri.pathname}${parsedUri.search}`; + } catch (e) { + // origin-form URI; keep as-is + } + } + + if (expectedIR.version === 'HTTP/2' || expectedIR.version === 'HTTP/2.0' || expectedIR.version === 'HTTP/3') { + expectedIR.version = serverIR.version; + } + if (expectedIR.headers) { expectedIR.headers = expectedIR.headers.map(h => ({ name: h.name.toLowerCase(), value: h.value })); expectedIR.headers = expectedIR.headers.filter(h => !IGNORED_HEADERS.includes(h.name)); diff --git a/test-fixtures/e2e/017-http-2-protocol.data.json b/test-fixtures/e2e/017-http-2-protocol.data.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test-fixtures/e2e/017-http-2-protocol.data.json @@ -0,0 +1 @@ +{} diff --git a/test-fixtures/e2e/017-http-2-protocol.httpt b/test-fixtures/e2e/017-http-2-protocol.httpt new file mode 100644 index 0000000..6c86b59 --- /dev/null +++ b/test-fixtures/e2e/017-http-2-protocol.httpt @@ -0,0 +1,3 @@ +GET /h2/resource HTTP/2 +Host: example.com + diff --git a/test-fixtures/e2e/017-http-2-protocol.httpt-ir b/test-fixtures/e2e/017-http-2-protocol.httpt-ir new file mode 100644 index 0000000..cbce30b --- /dev/null +++ b/test-fixtures/e2e/017-http-2-protocol.httpt-ir @@ -0,0 +1,8 @@ +{ + "schema-version": "1.0", + "method": "GET", + "host": "example.com", + "uri": "/h2/resource", + "version": "HTTP/2", + "headers": [] +} diff --git a/test-fixtures/e2e/017-http-2-protocol.httpt-map b/test-fixtures/e2e/017-http-2-protocol.httpt-map new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/test-fixtures/e2e/017-http-2-protocol.httpt-map @@ -0,0 +1 @@ +[] diff --git a/test-fixtures/e2e/017-http-2-protocol.httpt-r b/test-fixtures/e2e/017-http-2-protocol.httpt-r new file mode 100644 index 0000000..6c86b59 --- /dev/null +++ b/test-fixtures/e2e/017-http-2-protocol.httpt-r @@ -0,0 +1,3 @@ +GET /h2/resource HTTP/2 +Host: example.com + diff --git a/test-fixtures/e2e/018-bizarre-spacing.data.json b/test-fixtures/e2e/018-bizarre-spacing.data.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test-fixtures/e2e/018-bizarre-spacing.data.json @@ -0,0 +1 @@ +{} diff --git a/test-fixtures/e2e/018-bizarre-spacing.httpt b/test-fixtures/e2e/018-bizarre-spacing.httpt new file mode 100644 index 0000000..b1128aa --- /dev/null +++ b/test-fixtures/e2e/018-bizarre-spacing.httpt @@ -0,0 +1,3 @@ +GET /spaced?q=ok HTTP/1.1 +Host: example.com + diff --git a/test-fixtures/e2e/018-bizarre-spacing.httpt-ir b/test-fixtures/e2e/018-bizarre-spacing.httpt-ir new file mode 100644 index 0000000..a6d76a9 --- /dev/null +++ b/test-fixtures/e2e/018-bizarre-spacing.httpt-ir @@ -0,0 +1,8 @@ +{ + "schema-version": "1.0", + "method": "GET", + "host": "example.com", + "uri": "/spaced?q=ok", + "version": "HTTP/1.1", + "headers": [] +} diff --git a/test-fixtures/e2e/018-bizarre-spacing.httpt-map b/test-fixtures/e2e/018-bizarre-spacing.httpt-map new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/test-fixtures/e2e/018-bizarre-spacing.httpt-map @@ -0,0 +1 @@ +[] diff --git a/test-fixtures/e2e/018-bizarre-spacing.httpt-r b/test-fixtures/e2e/018-bizarre-spacing.httpt-r new file mode 100644 index 0000000..b1128aa --- /dev/null +++ b/test-fixtures/e2e/018-bizarre-spacing.httpt-r @@ -0,0 +1,3 @@ +GET /spaced?q=ok HTTP/1.1 +Host: example.com + diff --git a/test-fixtures/e2e/019-absolute-uri.data.json b/test-fixtures/e2e/019-absolute-uri.data.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test-fixtures/e2e/019-absolute-uri.data.json @@ -0,0 +1 @@ +{} diff --git a/test-fixtures/e2e/019-absolute-uri.httpt b/test-fixtures/e2e/019-absolute-uri.httpt new file mode 100644 index 0000000..51ed6db --- /dev/null +++ b/test-fixtures/e2e/019-absolute-uri.httpt @@ -0,0 +1,4 @@ +GET https://api.example.com/v1/resource?q=1 HTTP/1.1 +Host: api.example.com +Accept: application/json + diff --git a/test-fixtures/e2e/019-absolute-uri.httpt-ir b/test-fixtures/e2e/019-absolute-uri.httpt-ir new file mode 100644 index 0000000..bffdd58 --- /dev/null +++ b/test-fixtures/e2e/019-absolute-uri.httpt-ir @@ -0,0 +1,10 @@ +{ + "schema-version": "1.0", + "method": "GET", + "host": "api.example.com", + "uri": "https://api.example.com/v1/resource?q=1", + "version": "HTTP/1.1", + "headers": [ + { "name": "Accept", "value": "application/json" } + ] +} diff --git a/test-fixtures/e2e/019-absolute-uri.httpt-map b/test-fixtures/e2e/019-absolute-uri.httpt-map new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/test-fixtures/e2e/019-absolute-uri.httpt-map @@ -0,0 +1 @@ +[] diff --git a/test-fixtures/e2e/019-absolute-uri.httpt-r b/test-fixtures/e2e/019-absolute-uri.httpt-r new file mode 100644 index 0000000..51ed6db --- /dev/null +++ b/test-fixtures/e2e/019-absolute-uri.httpt-r @@ -0,0 +1,4 @@ +GET https://api.example.com/v1/resource?q=1 HTTP/1.1 +Host: api.example.com +Accept: application/json + diff --git a/test-fixtures/e2e/020-malformed-json.data.json b/test-fixtures/e2e/020-malformed-json.data.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test-fixtures/e2e/020-malformed-json.data.json @@ -0,0 +1 @@ +{} diff --git a/test-fixtures/e2e/020-malformed-json.error.json b/test-fixtures/e2e/020-malformed-json.error.json new file mode 100644 index 0000000..d837f47 --- /dev/null +++ b/test-fixtures/e2e/020-malformed-json.error.json @@ -0,0 +1,3 @@ +{ + "name": "TemplateSyntaxError" +} diff --git a/test-fixtures/e2e/020-malformed-json.httpt b/test-fixtures/e2e/020-malformed-json.httpt new file mode 100644 index 0000000..d517638 --- /dev/null +++ b/test-fixtures/e2e/020-malformed-json.httpt @@ -0,0 +1,5 @@ +POST /bad-json HTTP/1.1 +Host: example.com +:httpt-body-type: json + +{"broken": true diff --git a/test-fixtures/e2e/TEST_MATRIX.md b/test-fixtures/e2e/TEST_MATRIX.md index 5d5622a..658997a 100644 --- a/test-fixtures/e2e/TEST_MATRIX.md +++ b/test-fixtures/e2e/TEST_MATRIX.md @@ -4,26 +4,26 @@ This ledger maps our test fixtures to the 7 testing dimensions defined in the `R | Fixture ID | Target Dimension | Description | Status | | :--- | :--- | :--- | :--- | -| `001-simple-get` | 5. Source Mapping | Basic GET with URL and header injection. | ✅ Implemented | -| `002-post-json` | 4. JSON Nuances | Complex JSON payload with nested structures. | ✅ Implemented | -| `003-post-text-explicit` | 1. Body Type | Explicit `:httpt-body-type: text`. | ✅ Implemented | -| `004-post-json-native` | 1. Body Type | Explicit `:httpt-body-type: json` parsing. | ✅ Implemented | -| `005-post-base64` | 1. Body Type | Explicit `:httpt-body-type: base64` decoding. | ✅ Implemented | -| `006-post-provided` | 1. Body Type | Explicit `:httpt-body-type: provided` streaming. | ✅ Implemented | -| `007-conflict-provided-body` | 1. Body Type | Verifying shadowed body text is ignored. | ✅ Implemented | -| `008-boundary-unix-lf` | 2. Boundary | Parsing with `\n\n` instead of CRLF. | ✅ Implemented | +| `001-simple-get` | 5. Source Mapping | Basic GET with URL and header injection. | Implemented | +| `002-post-json` | 4. JSON Nuances | Complex JSON payload with nested structures. | Implemented | +| `003-post-text-explicit` | 1. Body Type | Explicit `:httpt-body-type: text`. | Implemented | +| `004-post-json-native` | 1. Body Type | Explicit `:httpt-body-type: json` parsing. | Implemented | +| `005-post-base64` | 1. Body Type | Explicit `:httpt-body-type: base64` decoding. | Implemented | +| `006-post-provided` | 1. Body Type | Explicit `:httpt-body-type: provided` streaming. | Implemented | +| `007-conflict-provided-body` | 1. Body Type | Verifying shadowed body text is ignored. | Implemented | +| `008-boundary-unix-lf` | 2. Boundary | Parsing with `\n\n` instead of CRLF. | Implemented | | **Phase 1: Dynamic Injection & Identity** | | | | -| `009-dynamic-headers` | 7. Dynamic Injection | Injecting `data.headers` before the boundary. | ✅ Implemented | -| `010-dynamic-body` | 7. Dynamic Injection | Injecting `data.body` stream and pseudo-header. | ✅ Implemented | -| `011-identity-template` | 7. Dynamic Injection | 2-line template parsing a full IR context. | ✅ Implemented | -| `012-error-body-conflict` | 7. Dynamic Injection | `BodyConflictError` when template has a body. | ✅ Implemented | +| `009-dynamic-headers` | 7. Dynamic Injection | Injecting `data.headers` before the boundary. | Implemented | +| `010-dynamic-body` | 7. Dynamic Injection | Injecting `data.body` stream and pseudo-header. | Implemented | +| `011-identity-template` | 7. Dynamic Injection | 2-line template parsing a full IR context. | Implemented | +| `012-error-body-conflict` | 7. Dynamic Injection | `BodyConflictError` when template has a body. | Implemented | | **Phase 2: Boundaries & Header Edge Cases** | | | | -| `013-minimalist-boundary` | 2. Boundary | Request line immediately followed by double-newline. | ✅ Implemented | -| `014-body-junk` | 2. Boundary | Body containing its own double-newlines. | ✅ Implemented | -| `015-multi-headers` | 3. Header Hygiene | Preserving duplicate headers (e.g., `Set-Cookie`). | ✅ Implemented | -| `016-empty-header` | 3. Header Hygiene | Headers with keys but no values. | ✅ Implemented | +| `013-minimalist-boundary` | 2. Boundary | Request line immediately followed by double-newline. | Implemented | +| `014-body-junk` | 2. Boundary | Body containing its own double-newlines. | Implemented | +| `015-multi-headers` | 3. Header Hygiene | Preserving duplicate headers (e.g., `Set-Cookie`). | Implemented | +| `016-empty-header` | 3. Header Hygiene | Headers with keys but no values. | Implemented | | **Phase 3: Protocol & JSON Quirks** | | | | -| `017-http-2-protocol` | 6. Protocol Edge Cases | Request line parsing with `HTTP/2`. | 🚧 Planned | -| `018-bizarre-spacing` | 6. Protocol Edge Cases | Handling extra spaces in the request line. | 🚧 Planned | -| `019-absolute-uri` | 6. Protocol Edge Cases | Handling absolute URIs in the request line. | 🚧 Planned | -| `020-malformed-json` | 4. JSON Nuances | Fallback/error handling for invalid JSON bodies. | 🚧 Planned | +| `017-http-2-protocol` | 6. Protocol Edge Cases | Request line parsing with `HTTP/2`. | Implemented | +| `018-bizarre-spacing` | 6. Protocol Edge Cases | Handling extra spaces in the request line. | Implemented | +| `019-absolute-uri` | 6. Protocol Edge Cases | Handling absolute URIs in the request line. | Implemented | +| `020-malformed-json` | 4. JSON Nuances | Error handling for invalid JSON bodies. | Implemented | From a2bd882fde149302181c9dc2ae8e50ffc74fe01c Mon Sep 17 00:00:00 2001 From: Waleed Ahmad <138612573+waleedm007@users.noreply.github.com> Date: Fri, 3 Jul 2026 12:32:04 +0500 Subject: [PATCH 5/8] Rewrite hydrate as cursor state machine --- js/packages/core/src/pipeline.js | 266 +++++++++++++++++++------ js/packages/core/test/pipeline.test.js | 9 + 2 files changed, 217 insertions(+), 58 deletions(-) diff --git a/js/packages/core/src/pipeline.js b/js/packages/core/src/pipeline.js index f7e6fd0..89a45ee 100644 --- a/js/packages/core/src/pipeline.js +++ b/js/packages/core/src/pipeline.js @@ -188,18 +188,39 @@ function materializeStream(value) { throw createNamedError('TemplateSyntaxError', 'Only string and Buffer stream values are supported synchronously'); } +function isHydrateFunctionName(value) { + if (!value) { + return false; + } + + for (const char of value) { + const code = char.charCodeAt(0); + const isDigit = code >= 48 && code <= 57; + const isUpper = code >= 65 && code <= 90; + const isLower = code >= 97 && code <= 122; + if (!isDigit && !isUpper && !isLower && char !== '-') { + return false; + } + } + + return true; +} + function parseHydrateFunction(functionSource) { const trimmed = functionSource.trim(); - const match = trimmed.match(/^([A-Za-z0-9-]+)(?:\((.*)\))?$/); + const openParen = trimmed.indexOf('('); + const closeParen = trimmed.endsWith(')') ? trimmed.length - 1 : -1; + const hasArguments = openParen !== -1; + const name = hasArguments ? trimmed.slice(0, openParen).trim() : trimmed; - if (!match) { + if (!isHydrateFunctionName(name) || (hasArguments && closeParen === -1)) { throw createNamedError('TemplateSyntaxError', `Invalid hydrate function: ${trimmed}`); } - const name = match[1]; - const args = match[2] == null || !match[2].trim() + const argumentSource = hasArguments ? trimmed.slice(openParen + 1, closeParen) : ''; + const args = !argumentSource.trim() ? [] - : match[2].split(',').map((arg) => arg.trim()).filter(Boolean); + : argumentSource.split(',').map((arg) => arg.trim()).filter(Boolean); if (!BUILT_IN_FUNCTIONS.has(name)) { throw createNamedError('TemplateSyntaxError', `Unsupported hydrate function: ${name}`); @@ -231,85 +252,214 @@ function applyHydrateFunction(value, hydrateFunction, streams) { } } +function parseHydrateTag(inner, data, streams) { + const parts = inner.split('|').map((part) => part.trim()).filter(Boolean); + if (parts.length < 2) { + throw createNamedError('TemplateSyntaxError', 'Template tag must include at least one function'); + } + + const dataKey = parts[0]; + if (!Object.prototype.hasOwnProperty.call(data, dataKey)) { + throw createNamedError('MissingArgumentError', `Missing data key: ${dataKey}`, { + missing: dataKey, + }); + } + + let replacement = data[dataKey]; + for (const transform of parts.slice(1).map(parseHydrateFunction)) { + replacement = applyHydrateFunction(replacement, transform, streams); + } + + return String(replacement); +} + function hydrate(template, data, streams = []) { const prepared = prepareHydrationContext(data, streams); data = prepared.data; streams = prepared.streams; - const map = []; const source = String(template); const newline = source.includes('\r\n') ? '\r\n' : '\n'; - let shift = 0; - - let resolved = source.replace(/\{\{\s*([^|}]+?)\s*((?:\|\s*[^|}]+?\s*)+)\}\}/g, (match, key, functionSource, offset) => { - const dataKey = key.trim(); - const transforms = functionSource - .split('|') - .map((part) => part.trim()) - .filter(Boolean) - .map(parseHydrateFunction); - - if (!Object.prototype.hasOwnProperty.call(data, dataKey)) { - throw createNamedError('MissingArgumentError', `Missing data key: ${dataKey}`, { - missing: dataKey, - }); + const map = []; + const output = []; + let writeCursor = 0; + let bodyStream = null; + let boundarySeen = false; + + const dynamicHeadLines = []; + if (Array.isArray(data.headers)) { + for (const { name, value } of data.headers) { + dynamicHeadLines.push(`${name}: ${value}`); } + } + + if (data.body) { + dynamicHeadLines.push(`:httpt-body-type: ${data.body.type || 'text'}`); + } - let replacement = data[dataKey]; - for (const transform of transforms) { - replacement = applyHydrateFunction(replacement, transform, streams); + function appendRaw(value, checkBoundary = true) { + for (const char of String(value)) { + output.push(char); + writeCursor += char.length; + + if (checkBoundary && !boundarySeen) { + const boundary = findBoundaryAtTail(); + if (boundary) { + enterBody(boundary); + } + } } - replacement = String(replacement); + } - map.push({ - 'hydrated-start': offset + shift, - 'original-start': offset, - 'hydrated-length': replacement.length, - 'original-length': match.length, - }); - shift += replacement.length - match.length; + function findBoundaryAtTail() { + const length = output.length; + if ( + length >= 4 && + output[length - 4] === '\r' && + output[length - 3] === '\n' && + output[length - 2] === '\r' && + output[length - 1] === '\n' + ) { + return '\r\n\r\n'; + } - return replacement; - }); + if (length >= 2 && output[length - 2] === '\n' && output[length - 1] === '\n') { + return '\n\n'; + } - const boundaryMatch = resolved.match(/\r?\n\r?\n/); - const boundaryIndex = boundaryMatch ? boundaryMatch.index : resolved.length; - let boundary = boundaryMatch ? boundaryMatch[0] : ''; - let head = resolved.slice(0, boundaryIndex).replace(/(?:\r?\n)+$/, ''); - let body = boundaryMatch ? resolved.slice(boundaryIndex + boundary.length) : ''; - let bodyStream = null; + return null; + } - if (Array.isArray(data.headers) && data.headers.length > 0) { - const dynamicHeaders = data.headers - .map(({ name, value }) => `${name}: ${value}`) - .join(newline); - head = `${head}${newline}${dynamicHeaders}`; + function trimTrailingNewlines() { + while (output.length > 0 && output[output.length - 1] === '\n') { + output.pop(); + writeCursor -= 1; + if (output[output.length - 1] === '\r') { + output.pop(); + writeCursor -= 1; + } + } } - if (data.body) { - if (body.trim()) { - const error = new Error('Template already contains a body'); - error.name = 'BodyConflictError'; - throw error; + function injectDynamicHeadLines() { + if (dynamicHeadLines.length === 0) { + return; } - const bodyType = data.body.type || 'text'; - head = `${head}${newline}:httpt-body-type: ${bodyType}`; + appendRaw(newline, false); + appendRaw(dynamicHeadLines.join(newline), false); + } + + function attachDynamicBody() { + if (!data.body) { + return; + } + + const bodyType = data.body.type || 'text'; if (bodyType === 'provided') { const streamIndex = data.body.content == null ? 0 : data.body.content; bodyStream = streams[streamIndex] || null; - body = ''; - } else { - body = data.body.content == null ? '' : String(data.body.content); - if (body && !boundary) { - boundary = `${newline}${newline}`; + return; + } + + if (data.body.content != null) { + appendRaw(String(data.body.content), false); + } + } + + function enterBody(boundary) { + boundarySeen = true; + output.length -= boundary.length; + writeCursor -= boundary.length; + injectDynamicHeadLines(); + appendRaw(boundary, false); + attachDynamicBody(); + } + + function finishHeadAtEof() { + trimTrailingNewlines(); + injectDynamicHeadLines(); + + if (!data.body) { + appendRaw(newline, false); + return; + } + + const bodyType = data.body.type || 'text'; + if (bodyType === 'provided') { + attachDynamicBody(); + appendRaw(newline, false); + return; + } + + appendRaw(`${newline}${newline}`, false); + attachDynamicBody(); + } + + function assertNoTemplateBodyCharacter(char) { + const isWhitespace = char === ' ' || char === '\t' || char === '\n' || char === '\r'; + if (data.body && !isWhitespace) { + const error = new Error('Template already contains a body'); + error.name = 'BodyConflictError'; + throw error; + } + } + + let readCursor = 0; + while (readCursor < source.length) { + const char = source[readCursor]; + + if (boundarySeen && data.body) { + assertNoTemplateBodyCharacter(char); + readCursor += 1; + continue; + } + + if (char === '{' && source[readCursor + 1] === '{') { + const tagStart = readCursor; + readCursor += 2; + let inner = ''; + + while (readCursor < source.length) { + if (source[readCursor] === '{' && source[readCursor + 1] === '{') { + throw createNamedError('TemplateSyntaxError', 'Nested template tags are not allowed', { index: readCursor }); + } + + if (source[readCursor] === '}' && source[readCursor + 1] === '}') { + break; + } + + inner += source[readCursor]; + readCursor += 1; + } + + if (readCursor >= source.length) { + throw createNamedError('TemplateSyntaxError', 'Unclosed template tag', { index: tagStart }); } + + const originalLength = readCursor + 2 - tagStart; + const hydratedStart = writeCursor; + const replacement = parseHydrateTag(inner, data, streams); + map.push({ + 'hydrated-start': hydratedStart, + 'original-start': tagStart, + 'hydrated-length': replacement.length, + 'original-length': originalLength, + }); + appendRaw(replacement); + readCursor += 2; + continue; } + + appendRaw(char); + readCursor += 1; } - resolved = `${head}${boundary || newline}${body}`; + if (!boundarySeen) { + finishHeadAtEof(); + } - return { resolved, map, bodyStream }; + return { resolved: output.join(''), map, bodyStream }; } async function hydrateAsync(template, data, streams = []) { diff --git a/js/packages/core/test/pipeline.test.js b/js/packages/core/test/pipeline.test.js index e4607ee..06b36c0 100644 --- a/js/packages/core/test/pipeline.test.js +++ b/js/packages/core/test/pipeline.test.js @@ -54,4 +54,13 @@ describe('Pipeline: Hydrate & Parse', () => { assert.strictEqual(parsedBodyStream, stream); assert.deepEqual(ir.body, { type: 'provided', content: 0 }); }); + + it('should reject template body content before resolving body placeholders when data.body is provided', () => { + const template = 'POST /upload HTTP/1.1\nHost: example.com\n\n{{ missing | raw }}'; + + assert.throws( + () => hydrate(template, { body: { type: 'text', content: 'dynamic body' } }), + (error) => error.name === 'BodyConflictError' + ); + }); }); From a1308da925647255d83da07c64a538551ff72733 Mon Sep 17 00:00:00 2001 From: Waleed Ahmad <138612573+waleedm007@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:22:22 +0500 Subject: [PATCH 6/8] Stream hydrate templates through state machine --- js/packages/core/src/pipeline.js | 254 ++++++++++++++---- js/packages/core/test/pipeline.test.js | 31 ++- test-fixtures/e2e/010-dynamic-body.httpt-r | 1 + .../e2e/011-identity-template.httpt-r | 1 + 4 files changed, 227 insertions(+), 60 deletions(-) diff --git a/js/packages/core/src/pipeline.js b/js/packages/core/src/pipeline.js index 89a45ee..a77c10c 100644 --- a/js/packages/core/src/pipeline.js +++ b/js/packages/core/src/pipeline.js @@ -273,18 +273,27 @@ function parseHydrateTag(inner, data, streams) { return String(replacement); } -function hydrate(template, data, streams = []) { +function createHydrationState(data, streams, sink) { const prepared = prepareHydrationContext(data, streams); data = prepared.data; streams = prepared.streams; - const source = String(template); - const newline = source.includes('\r\n') ? '\r\n' : '\n'; const map = []; - const output = []; + let newline = '\n'; let writeCursor = 0; + let readCursor = 0; let bodyStream = null; let boundarySeen = false; + let pendingHead = ''; + let mode = 'text'; + let pendingSourceOpen = false; + let pendingSourceOpenIndex = -1; + let tagStart = -1; + let tagInner = ''; + let tagPendingOpen = false; + let tagPendingOpenIndex = -1; + let tagPendingClose = false; + let previousSourceChar = ''; const dynamicHeadLines = []; if (Array.isArray(data.headers)) { @@ -297,45 +306,55 @@ function hydrate(template, data, streams = []) { dynamicHeadLines.push(`:httpt-body-type: ${data.body.type || 'text'}`); } + function emit(value) { + if (value) { + sink.write(value); + } + } + function appendRaw(value, checkBoundary = true) { for (const char of String(value)) { - output.push(char); writeCursor += char.length; if (checkBoundary && !boundarySeen) { + pendingHead += char; const boundary = findBoundaryAtTail(); if (boundary) { enterBody(boundary); + } else { + flushSafeHeadPrefix(); } + } else { + emit(char); } } } function findBoundaryAtTail() { - const length = output.length; - if ( - length >= 4 && - output[length - 4] === '\r' && - output[length - 3] === '\n' && - output[length - 2] === '\r' && - output[length - 1] === '\n' - ) { + if (pendingHead.endsWith('\r\n\r\n')) { return '\r\n\r\n'; } - if (length >= 2 && output[length - 2] === '\n' && output[length - 1] === '\n') { + if (pendingHead.endsWith('\n\n')) { return '\n\n'; } return null; } + function flushSafeHeadPrefix() { + while (pendingHead.length > 4) { + emit(pendingHead[0]); + pendingHead = pendingHead.slice(1); + } + } + function trimTrailingNewlines() { - while (output.length > 0 && output[output.length - 1] === '\n') { - output.pop(); + while (pendingHead.endsWith('\n')) { + pendingHead = pendingHead.slice(0, -1); writeCursor -= 1; - if (output[output.length - 1] === '\r') { - output.pop(); + if (pendingHead.endsWith('\r')) { + pendingHead = pendingHead.slice(0, -1); writeCursor -= 1; } } @@ -369,7 +388,9 @@ function hydrate(template, data, streams = []) { function enterBody(boundary) { boundarySeen = true; - output.length -= boundary.length; + const headPrefix = pendingHead.slice(0, -boundary.length); + emit(headPrefix); + pendingHead = ''; writeCursor -= boundary.length; injectDynamicHeadLines(); appendRaw(boundary, false); @@ -378,6 +399,8 @@ function hydrate(template, data, streams = []) { function finishHeadAtEof() { trimTrailingNewlines(); + emit(pendingHead); + pendingHead = ''; injectDynamicHeadLines(); if (!data.body) { @@ -388,7 +411,7 @@ function hydrate(template, data, streams = []) { const bodyType = data.body.type || 'text'; if (bodyType === 'provided') { attachDynamicBody(); - appendRaw(newline, false); + appendRaw(`${newline}${newline}`, false); return; } @@ -405,65 +428,180 @@ function hydrate(template, data, streams = []) { } } - let readCursor = 0; - while (readCursor < source.length) { - const char = source[readCursor]; - + function processTextChar(char, index) { if (boundarySeen && data.body) { assertNoTemplateBodyCharacter(char); - readCursor += 1; - continue; + return; } - if (char === '{' && source[readCursor + 1] === '{') { - const tagStart = readCursor; - readCursor += 2; - let inner = ''; + if (pendingSourceOpen) { + if (char === '{') { + mode = 'tag'; + tagStart = pendingSourceOpenIndex; + tagInner = ''; + pendingSourceOpen = false; + return; + } - while (readCursor < source.length) { - if (source[readCursor] === '{' && source[readCursor + 1] === '{') { - throw createNamedError('TemplateSyntaxError', 'Nested template tags are not allowed', { index: readCursor }); - } + appendRaw('{'); + pendingSourceOpen = false; + pendingSourceOpenIndex = -1; + processTextChar(char, index); + return; + } - if (source[readCursor] === '}' && source[readCursor + 1] === '}') { - break; - } + if (char === '{') { + pendingSourceOpen = true; + pendingSourceOpenIndex = index; + return; + } - inner += source[readCursor]; - readCursor += 1; + appendRaw(char); + } + + function processTagChar(char, index) { + if (tagPendingOpen) { + if (char === '{') { + throw createNamedError('TemplateSyntaxError', 'Nested template tags are not allowed', { index: tagPendingOpenIndex }); } + tagInner += '{'; + tagPendingOpen = false; + tagPendingOpenIndex = -1; + processTagChar(char, index); + return; + } - if (readCursor >= source.length) { - throw createNamedError('TemplateSyntaxError', 'Unclosed template tag', { index: tagStart }); + if (tagPendingClose) { + if (char === '}') { + const originalLength = index + 1 - tagStart; + const hydratedStart = writeCursor; + const replacement = parseHydrateTag(tagInner, data, streams); + map.push({ + 'hydrated-start': hydratedStart, + 'original-start': tagStart, + 'hydrated-length': replacement.length, + 'original-length': originalLength, + }); + appendRaw(replacement); + mode = 'text'; + tagStart = -1; + tagInner = ''; + tagPendingClose = false; + return; } - const originalLength = readCursor + 2 - tagStart; - const hydratedStart = writeCursor; - const replacement = parseHydrateTag(inner, data, streams); - map.push({ - 'hydrated-start': hydratedStart, - 'original-start': tagStart, - 'hydrated-length': replacement.length, - 'original-length': originalLength, - }); - appendRaw(replacement); - readCursor += 2; - continue; + tagInner += '}'; + tagPendingClose = false; + processTagChar(char, index); + return; } - appendRaw(char); - readCursor += 1; + if (char === '{') { + tagPendingOpen = true; + tagPendingOpenIndex = index; + return; + } + + if (char === '}') { + tagPendingClose = true; + return; + } + + tagInner += char; + } + + function feedChar(char) { + const index = readCursor; + readCursor += char.length; + + if (char === '\n' && previousSourceChar === '\r') { + newline = '\r\n'; + } + + if (mode === 'tag') { + processTagChar(char, index); + } else { + processTextChar(char, index); + } + + previousSourceChar = char; } - if (!boundarySeen) { - finishHeadAtEof(); + function finish() { + if (mode === 'tag') { + throw createNamedError('TemplateSyntaxError', 'Unclosed template tag', { index: tagStart }); + } + + if (pendingSourceOpen) { + appendRaw('{'); + pendingSourceOpen = false; + } + + if (!boundarySeen) { + finishHeadAtEof(); + } else if (pendingHead) { + emit(pendingHead); + pendingHead = ''; + } + + return { map, bodyStream }; } + return { feedChar, finish }; +} + +function hydrate(template, data, streams = []) { + const output = []; + const state = createHydrationState(data, streams, { + write(value) { + output.push(value); + }, + }); + + for (const char of String(template)) { + state.feedChar(char); + } + + const { map, bodyStream } = state.finish(); return { resolved: output.join(''), map, bodyStream }; } async function hydrateAsync(template, data, streams = []) { - return hydrate(await resolveToString(template), data, streams); + if (typeof template === 'string' || Buffer.isBuffer(template) || template instanceof Uint8Array) { + return hydrate(await resolveToString(template), data, streams); + } + + const output = []; + const state = createHydrationState(data, streams, { + write(value) { + output.push(value); + }, + }); + + if (template && typeof template.getReader === 'function') { + const reader = template.getReader(); + while (true) { + const { value, done } = await reader.read(); + if (done) break; + for (const char of Buffer.from(value).toString('utf8')) { + state.feedChar(char); + } + } + } else if (template && typeof template[Symbol.asyncIterator] === 'function') { + for await (const chunk of template) { + const text = Buffer.isBuffer(chunk) || chunk instanceof Uint8Array + ? Buffer.from(chunk).toString('utf8') + : String(chunk); + for (const char of text) { + state.feedChar(char); + } + } + } else { + return hydrate(await resolveToString(template), data, streams); + } + + const { map, bodyStream } = state.finish(); + return { resolved: output.join(''), map, bodyStream }; } function parse(resolved, optionalBodyStream = null) { diff --git a/js/packages/core/test/pipeline.test.js b/js/packages/core/test/pipeline.test.js index 06b36c0..f72d32b 100644 --- a/js/packages/core/test/pipeline.test.js +++ b/js/packages/core/test/pipeline.test.js @@ -1,6 +1,6 @@ const { describe, it } = require('node:test'); const assert = require('node:assert'); -const { hydrate, parse } = require('../src/pipeline.js'); +const { hydrate, hydrateAsync, parse } = require('../src/pipeline.js'); const { loadE2eFixtures } = require('@httpt/test-utils'); function normalizeLineEndings(value) { @@ -50,11 +50,38 @@ describe('Pipeline: Hydrate & Parse', () => { const { resolved, bodyStream } = hydrate(template, { body: { type: 'provided', content: 0 } }, [stream]); const { ir, bodyStream: parsedBodyStream } = parse(resolved, bodyStream); - assert.equal(resolved.endsWith('\r\n:httpt-body-type: provided\r\n'), true); + assert.equal(resolved.endsWith('\r\n:httpt-body-type: provided\r\n\r\n'), true); assert.strictEqual(parsedBodyStream, stream); assert.deepEqual(ir.body, { type: 'provided', content: 0 }); }); + it('should hydrate streamed template input across chunk boundaries', async () => { + async function* chunks() { + const source = 'GET /{{ path | url }} HTTP/1.1\r\nHost: {{ host | raw }}\r\n'; + for (const char of source) { + yield Buffer.from(char); + } + } + + const { resolved, map } = await hydrateAsync(chunks(), { path: 'a b', host: 'example.com' }); + + assert.equal(resolved, 'GET /a%20b HTTP/1.1\r\nHost: example.com\r\n'); + assert.deepEqual(map, [ + { + 'hydrated-start': 5, + 'original-start': 5, + 'hydrated-length': 5, + 'original-length': 16, + }, + { + 'hydrated-start': 27, + 'original-start': 38, + 'hydrated-length': 11, + 'original-length': 16, + }, + ]); + }); + it('should reject template body content before resolving body placeholders when data.body is provided', () => { const template = 'POST /upload HTTP/1.1\nHost: example.com\n\n{{ missing | raw }}'; diff --git a/test-fixtures/e2e/010-dynamic-body.httpt-r b/test-fixtures/e2e/010-dynamic-body.httpt-r index c683c5d..e59338d 100644 --- a/test-fixtures/e2e/010-dynamic-body.httpt-r +++ b/test-fixtures/e2e/010-dynamic-body.httpt-r @@ -2,3 +2,4 @@ POST /api/upload HTTP/1.1 Host: api.example.com Content-Type: text/plain :httpt-body-type: provided + diff --git a/test-fixtures/e2e/011-identity-template.httpt-r b/test-fixtures/e2e/011-identity-template.httpt-r index 9d95729..8627a0f 100644 --- a/test-fixtures/e2e/011-identity-template.httpt-r +++ b/test-fixtures/e2e/011-identity-template.httpt-r @@ -2,3 +2,4 @@ PATCH /api/identity HTTP/1.1 Host: identity.example.com Content-Type: application/json :httpt-body-type: provided + From 1f40a5069a2c1a0ef264c3a5a43d1ba5176a47ec Mon Sep 17 00:00:00 2001 From: Waleed Ahmad <138612573+waleedm007@users.noreply.github.com> Date: Sat, 4 Jul 2026 12:42:53 +0500 Subject: [PATCH 7/8] Simplify hydrate state machine --- js/packages/core/src/pipeline.js | 254 ++++++++++++++----------------- 1 file changed, 112 insertions(+), 142 deletions(-) diff --git a/js/packages/core/src/pipeline.js b/js/packages/core/src/pipeline.js index a77c10c..addac83 100644 --- a/js/packages/core/src/pipeline.js +++ b/js/packages/core/src/pipeline.js @@ -279,31 +279,28 @@ function createHydrationState(data, streams, sink) { streams = prepared.streams; const map = []; + const headLines = []; let newline = '\n'; - let writeCursor = 0; let readCursor = 0; + let writeCursor = 0; let bodyStream = null; - let boundarySeen = false; - let pendingHead = ''; - let mode = 'text'; - let pendingSourceOpen = false; - let pendingSourceOpenIndex = -1; - let tagStart = -1; - let tagInner = ''; - let tagPendingOpen = false; - let tagPendingOpenIndex = -1; - let tagPendingClose = false; - let previousSourceChar = ''; - - const dynamicHeadLines = []; + let inBody = false; + let headTail = ''; + let sourceOpenAt = null; + let tagOpenAt = null; + let tagClosePending = false; + let tagOpenPending = null; + let tagText = ''; + let previousChar = ''; + if (Array.isArray(data.headers)) { for (const { name, value } of data.headers) { - dynamicHeadLines.push(`${name}: ${value}`); + headLines.push(`${name}: ${value}`); } } if (data.body) { - dynamicHeadLines.push(`:httpt-body-type: ${data.body.type || 'text'}`); + headLines.push(`:httpt-body-type: ${data.body.type || 'text'}`); } function emit(value) { @@ -312,17 +309,36 @@ function createHydrationState(data, streams, sink) { } } - function appendRaw(value, checkBoundary = true) { + function boundaryAtEnd() { + if (headTail.endsWith('\r\n\r\n')) return '\r\n\r\n'; + if (headTail.endsWith('\n\n')) return '\n\n'; + return null; + } + + function emitSafeHead() { + while (headTail.length > 4) { + emit(headTail[0]); + headTail = headTail.slice(1); + } + } + + function write(value, checkBoundary = true) { for (const char of String(value)) { writeCursor += char.length; - if (checkBoundary && !boundarySeen) { - pendingHead += char; - const boundary = findBoundaryAtTail(); + if (checkBoundary && !inBody) { + headTail += char; + const boundary = boundaryAtEnd(); if (boundary) { - enterBody(boundary); + inBody = true; + emit(headTail.slice(0, -boundary.length)); + headTail = ''; + writeCursor -= boundary.length; + writeDynamicHead(); + write(boundary, false); + attachDynamicBody(); } else { - flushSafeHeadPrefix(); + emitSafeHead(); } } else { emit(char); @@ -330,49 +346,25 @@ function createHydrationState(data, streams, sink) { } } - function findBoundaryAtTail() { - if (pendingHead.endsWith('\r\n\r\n')) { - return '\r\n\r\n'; - } - - if (pendingHead.endsWith('\n\n')) { - return '\n\n'; - } - - return null; - } - - function flushSafeHeadPrefix() { - while (pendingHead.length > 4) { - emit(pendingHead[0]); - pendingHead = pendingHead.slice(1); - } - } - - function trimTrailingNewlines() { - while (pendingHead.endsWith('\n')) { - pendingHead = pendingHead.slice(0, -1); + function trimHeadEnd() { + while (headTail.endsWith('\n')) { + headTail = headTail.slice(0, -1); writeCursor -= 1; - if (pendingHead.endsWith('\r')) { - pendingHead = pendingHead.slice(0, -1); + if (headTail.endsWith('\r')) { + headTail = headTail.slice(0, -1); writeCursor -= 1; } } } - function injectDynamicHeadLines() { - if (dynamicHeadLines.length === 0) { - return; - } - - appendRaw(newline, false); - appendRaw(dynamicHeadLines.join(newline), false); + function writeDynamicHead() { + if (headLines.length === 0) return; + write(newline, false); + write(headLines.join(newline), false); } function attachDynamicBody() { - if (!data.body) { - return; - } + if (!data.body) return; const bodyType = data.body.type || 'text'; if (bodyType === 'provided') { @@ -382,44 +374,26 @@ function createHydrationState(data, streams, sink) { } if (data.body.content != null) { - appendRaw(String(data.body.content), false); + write(String(data.body.content), false); } } - function enterBody(boundary) { - boundarySeen = true; - const headPrefix = pendingHead.slice(0, -boundary.length); - emit(headPrefix); - pendingHead = ''; - writeCursor -= boundary.length; - injectDynamicHeadLines(); - appendRaw(boundary, false); - attachDynamicBody(); - } - - function finishHeadAtEof() { - trimTrailingNewlines(); - emit(pendingHead); - pendingHead = ''; - injectDynamicHeadLines(); + function finishHead() { + trimHeadEnd(); + emit(headTail); + headTail = ''; + writeDynamicHead(); if (!data.body) { - appendRaw(newline, false); + write(newline, false); return; } - const bodyType = data.body.type || 'text'; - if (bodyType === 'provided') { - attachDynamicBody(); - appendRaw(`${newline}${newline}`, false); - return; - } - - appendRaw(`${newline}${newline}`, false); + write(`${newline}${newline}`, false); attachDynamicBody(); } - function assertNoTemplateBodyCharacter(char) { + function rejectDynamicBodyConflict(char) { const isWhitespace = char === ' ' || char === '\t' || char === '\n' || char === '\r'; if (data.body && !isWhitespace) { const error = new Error('Template already contains a body'); @@ -428,120 +402,116 @@ function createHydrationState(data, streams, sink) { } } - function processTextChar(char, index) { - if (boundarySeen && data.body) { - assertNoTemplateBodyCharacter(char); + function closeTag(endIndex) { + const replacement = parseHydrateTag(tagText, data, streams); + map.push({ + 'hydrated-start': writeCursor, + 'original-start': tagOpenAt, + 'hydrated-length': replacement.length, + 'original-length': endIndex + 1 - tagOpenAt, + }); + + write(replacement); + tagOpenAt = null; + tagText = ''; + tagClosePending = false; + } + + function readText(char, index) { + if (inBody && data.body) { + rejectDynamicBodyConflict(char); return; } - if (pendingSourceOpen) { + if (sourceOpenAt != null) { if (char === '{') { - mode = 'tag'; - tagStart = pendingSourceOpenIndex; - tagInner = ''; - pendingSourceOpen = false; + tagOpenAt = sourceOpenAt; + sourceOpenAt = null; return; } - appendRaw('{'); - pendingSourceOpen = false; - pendingSourceOpenIndex = -1; - processTextChar(char, index); + write('{'); + sourceOpenAt = null; + readText(char, index); return; } if (char === '{') { - pendingSourceOpen = true; - pendingSourceOpenIndex = index; + sourceOpenAt = index; return; } - appendRaw(char); + write(char); } - function processTagChar(char, index) { - if (tagPendingOpen) { + function readTag(char, index) { + if (tagOpenPending != null) { if (char === '{') { - throw createNamedError('TemplateSyntaxError', 'Nested template tags are not allowed', { index: tagPendingOpenIndex }); + throw createNamedError('TemplateSyntaxError', 'Nested template tags are not allowed', { index: tagOpenPending }); } - tagInner += '{'; - tagPendingOpen = false; - tagPendingOpenIndex = -1; - processTagChar(char, index); + tagText += '{'; + tagOpenPending = null; + readTag(char, index); return; } - if (tagPendingClose) { + if (tagClosePending) { if (char === '}') { - const originalLength = index + 1 - tagStart; - const hydratedStart = writeCursor; - const replacement = parseHydrateTag(tagInner, data, streams); - map.push({ - 'hydrated-start': hydratedStart, - 'original-start': tagStart, - 'hydrated-length': replacement.length, - 'original-length': originalLength, - }); - appendRaw(replacement); - mode = 'text'; - tagStart = -1; - tagInner = ''; - tagPendingClose = false; + closeTag(index); return; } - tagInner += '}'; - tagPendingClose = false; - processTagChar(char, index); + tagText += '}'; + tagClosePending = false; + readTag(char, index); return; } if (char === '{') { - tagPendingOpen = true; - tagPendingOpenIndex = index; + tagOpenPending = index; return; } if (char === '}') { - tagPendingClose = true; + tagClosePending = true; return; } - tagInner += char; + tagText += char; } function feedChar(char) { const index = readCursor; readCursor += char.length; - if (char === '\n' && previousSourceChar === '\r') { + if (char === '\n' && previousChar === '\r') { newline = '\r\n'; } - if (mode === 'tag') { - processTagChar(char, index); + if (tagOpenAt == null) { + readText(char, index); } else { - processTextChar(char, index); + readTag(char, index); } - previousSourceChar = char; + previousChar = char; } function finish() { - if (mode === 'tag') { - throw createNamedError('TemplateSyntaxError', 'Unclosed template tag', { index: tagStart }); + if (tagOpenAt != null) { + throw createNamedError('TemplateSyntaxError', 'Unclosed template tag', { index: tagOpenAt }); } - if (pendingSourceOpen) { - appendRaw('{'); - pendingSourceOpen = false; + if (sourceOpenAt != null) { + write('{'); + sourceOpenAt = null; } - if (!boundarySeen) { - finishHeadAtEof(); - } else if (pendingHead) { - emit(pendingHead); - pendingHead = ''; + if (!inBody) { + finishHead(); + } else if (headTail) { + emit(headTail); + headTail = ''; } return { map, bodyStream }; From b8b9bf3a742dc27e5d8cf34d34beeb60a55d94f9 Mon Sep 17 00:00:00 2001 From: Waleed Ahmad <138612573+waleedm007@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:39:01 +0500 Subject: [PATCH 8/8] Clarify hydrate state names --- js/packages/core/src/pipeline.js | 108 +++++++++++++++---------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/js/packages/core/src/pipeline.js b/js/packages/core/src/pipeline.js index addac83..0b695cf 100644 --- a/js/packages/core/src/pipeline.js +++ b/js/packages/core/src/pipeline.js @@ -284,13 +284,13 @@ function createHydrationState(data, streams, sink) { let readCursor = 0; let writeCursor = 0; let bodyStream = null; - let inBody = false; - let headTail = ''; - let sourceOpenAt = null; - let tagOpenAt = null; - let tagClosePending = false; - let tagOpenPending = null; - let tagText = ''; + let bodyStarted = false; + let headBuffer = ''; + let pendingOpenBraceAt = null; + let tagStart = null; + let sawClosingBrace = false; + let nestedOpenBraceAt = null; + let tagContent = ''; let previousChar = ''; if (Array.isArray(data.headers)) { @@ -310,15 +310,15 @@ function createHydrationState(data, streams, sink) { } function boundaryAtEnd() { - if (headTail.endsWith('\r\n\r\n')) return '\r\n\r\n'; - if (headTail.endsWith('\n\n')) return '\n\n'; + if (headBuffer.endsWith('\r\n\r\n')) return '\r\n\r\n'; + if (headBuffer.endsWith('\n\n')) return '\n\n'; return null; } function emitSafeHead() { - while (headTail.length > 4) { - emit(headTail[0]); - headTail = headTail.slice(1); + while (headBuffer.length > 4) { + emit(headBuffer[0]); + headBuffer = headBuffer.slice(1); } } @@ -326,13 +326,13 @@ function createHydrationState(data, streams, sink) { for (const char of String(value)) { writeCursor += char.length; - if (checkBoundary && !inBody) { - headTail += char; + if (checkBoundary && !bodyStarted) { + headBuffer += char; const boundary = boundaryAtEnd(); if (boundary) { - inBody = true; - emit(headTail.slice(0, -boundary.length)); - headTail = ''; + bodyStarted = true; + emit(headBuffer.slice(0, -boundary.length)); + headBuffer = ''; writeCursor -= boundary.length; writeDynamicHead(); write(boundary, false); @@ -347,11 +347,11 @@ function createHydrationState(data, streams, sink) { } function trimHeadEnd() { - while (headTail.endsWith('\n')) { - headTail = headTail.slice(0, -1); + while (headBuffer.endsWith('\n')) { + headBuffer = headBuffer.slice(0, -1); writeCursor -= 1; - if (headTail.endsWith('\r')) { - headTail = headTail.slice(0, -1); + if (headBuffer.endsWith('\r')) { + headBuffer = headBuffer.slice(0, -1); writeCursor -= 1; } } @@ -380,8 +380,8 @@ function createHydrationState(data, streams, sink) { function finishHead() { trimHeadEnd(); - emit(headTail); - headTail = ''; + emit(headBuffer); + headBuffer = ''; writeDynamicHead(); if (!data.body) { @@ -403,41 +403,41 @@ function createHydrationState(data, streams, sink) { } function closeTag(endIndex) { - const replacement = parseHydrateTag(tagText, data, streams); + const replacement = parseHydrateTag(tagContent, data, streams); map.push({ 'hydrated-start': writeCursor, - 'original-start': tagOpenAt, + 'original-start': tagStart, 'hydrated-length': replacement.length, - 'original-length': endIndex + 1 - tagOpenAt, + 'original-length': endIndex + 1 - tagStart, }); write(replacement); - tagOpenAt = null; - tagText = ''; - tagClosePending = false; + tagStart = null; + tagContent = ''; + sawClosingBrace = false; } function readText(char, index) { - if (inBody && data.body) { + if (bodyStarted && data.body) { rejectDynamicBodyConflict(char); return; } - if (sourceOpenAt != null) { + if (pendingOpenBraceAt != null) { if (char === '{') { - tagOpenAt = sourceOpenAt; - sourceOpenAt = null; + tagStart = pendingOpenBraceAt; + pendingOpenBraceAt = null; return; } write('{'); - sourceOpenAt = null; + pendingOpenBraceAt = null; readText(char, index); return; } if (char === '{') { - sourceOpenAt = index; + pendingOpenBraceAt = index; return; } @@ -445,39 +445,39 @@ function createHydrationState(data, streams, sink) { } function readTag(char, index) { - if (tagOpenPending != null) { + if (nestedOpenBraceAt != null) { if (char === '{') { - throw createNamedError('TemplateSyntaxError', 'Nested template tags are not allowed', { index: tagOpenPending }); + throw createNamedError('TemplateSyntaxError', 'Nested template tags are not allowed', { index: nestedOpenBraceAt }); } - tagText += '{'; - tagOpenPending = null; + tagContent += '{'; + nestedOpenBraceAt = null; readTag(char, index); return; } - if (tagClosePending) { + if (sawClosingBrace) { if (char === '}') { closeTag(index); return; } - tagText += '}'; - tagClosePending = false; + tagContent += '}'; + sawClosingBrace = false; readTag(char, index); return; } if (char === '{') { - tagOpenPending = index; + nestedOpenBraceAt = index; return; } if (char === '}') { - tagClosePending = true; + sawClosingBrace = true; return; } - tagText += char; + tagContent += char; } function feedChar(char) { @@ -488,7 +488,7 @@ function createHydrationState(data, streams, sink) { newline = '\r\n'; } - if (tagOpenAt == null) { + if (tagStart == null) { readText(char, index); } else { readTag(char, index); @@ -498,20 +498,20 @@ function createHydrationState(data, streams, sink) { } function finish() { - if (tagOpenAt != null) { - throw createNamedError('TemplateSyntaxError', 'Unclosed template tag', { index: tagOpenAt }); + if (tagStart != null) { + throw createNamedError('TemplateSyntaxError', 'Unclosed template tag', { index: tagStart }); } - if (sourceOpenAt != null) { + if (pendingOpenBraceAt != null) { write('{'); - sourceOpenAt = null; + pendingOpenBraceAt = null; } - if (!inBody) { + if (!bodyStarted) { finishHead(); - } else if (headTail) { - emit(headTail); - headTail = ''; + } else if (headBuffer) { + emit(headBuffer); + headBuffer = ''; } return { map, bodyStream };