CLI: add --source-url flag for site-specific extraction on stdin/file input

src/cli.ts

@@ -19,6 +19,7 @@ export interface ParseOptions {
 	lang?: string;
 	userAgent?: string;
 	frontmatter?: boolean;
+	sourceUrl?: string;
 }
 
 interface ParseResult {
@@ -61,7 +62,7 @@ export async function parseSource(source: string | undefined, options: ParseOpti
 	};
 
 	let html: string;
-	let url: string | undefined;
+	let url: string | undefined = options.sourceUrl;
 
 	const usesStdin = !source || source === '-';
 	const isUrl = !usesStdin && (source.startsWith('http://') || source.startsWith('https://'));
@@ -72,6 +73,9 @@ export async function parseSource(source: string | undefined, options: ParseOpti
 		}
 		html = await readStdin(input);
 	} else if (isUrl) {
+		// Positional URL is also the content URL; --source-url, if any, has been
+		// pre-seeded into `url` above and is overwritten here intentionally so the
+		// fetched URL is authoritative for a live fetch.
 		url = source;
 		const initialUA = options.userAgent || getInitialUA(source);
 		html = await fetchPage(source, initialUA, options.lang);
@@ -174,6 +178,7 @@ export function createProgram(): Command {
 		.option('--debug', 'Enable debug mode')
 		.option('-l, --lang <code>', 'Preferred language (BCP 47, e.g. en, fr, ja)')
 		.option('-u, --user-agent <string>', 'Custom User-Agent header for HTTP requests (helps with 403/FORBIDDEN responses)')
+		.option('--source-url <url>', 'URL the input HTML originated from (enables site-specific extractors when source is stdin or a local file)')
 		.action(async (source: string | undefined, options: ParseOptions) => {
 			try {
 				const { output } = await parseSource(source, options);

tests/cli.test.ts

@@ -105,4 +105,49 @@ describe('CLI parseSource', () => {
 		// commander camelCases --user-agent → options.userAgent, which parseSource reads.
 		expect(option?.attributeName()).toBe('userAgent');
 	});
+
+	test('registers the --source-url flag', () => {
+		const parseCommand = createProgram().commands.find((c) => c.name() === 'parse');
+		const option = parseCommand?.options.find((o) => o.long === '--source-url');
+
+		expect(option).toBeDefined();
+		// commander camelCases --source-url → options.sourceUrl, which parseSource reads.
+		expect(option?.attributeName()).toBe('sourceUrl');
+	});
+
+	test('uses --source-url to activate site-specific extractors for stdin input', async () => {
+		// The Claude conversation fixture has data-testid="user-message" and
+		// .font-claude-response divs, which the ClaudeExtractor recognises and
+		// formats with **You** / **Claude** author labels — but only when a
+		// claude.ai URL is supplied as the document URL. Without a URL, the
+		// extractor registry has no domain match, generic scoring runs, and
+		// the author labels are absent from the output.
+		const claudeFixture = readFileSync(
+			join(__dirname, 'fixtures', 'extractor--claude-conversation.html'),
+			'utf-8'
+		);
+
+		// Baseline: without --source-url, ClaudeExtractor does not run, so the
+		// "**You**" / "**Claude**" author labels (only the conversation
+		// extractor emits these) are missing from the output.
+		const baseline = await parseSource(
+			'-',
+			{ markdown: true },
+			createMockStdin(claudeFixture)
+		);
+		expect(baseline.output).not.toContain('**You**');
+		expect(baseline.output).not.toContain('**Claude**');
+
+		// With --source-url pointing at claude.ai, ClaudeExtractor activates
+		// and emits the structured conversation turns with author labels.
+		const result = await parseSource(
+			'-',
+			{ markdown: true, sourceUrl: 'https://claude.ai/chat/example-fixture' },
+			createMockStdin(claudeFixture)
+		);
+		expect(result.output).toContain('**You**');
+		expect(result.output).toContain('**Claude**');
+		expect(result.output).toContain('Paris');
+		expect(result.output).toContain('Berlin');
+	});
 });

tests/expected/extractor--claude-conversation.md

@@ -0,0 +1,30 @@
+```json
+{
+  "title": "Capital cities — Claude",
+  "author": "",
+  "site": "Claude",
+  "published": ""
+}
+```
+
+**You**
+
+What is the capital of France?
+
+---
+
+**Claude**
+
+The capital of France is Paris.
+
+---
+
+**You**
+
+And Germany?
+
+---
+
+**Claude**
+
+The capital of Germany is Berlin.

tests/fixtures/extractor--claude-conversation.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<!-- {"url":"https://claude.ai/chat/example-fixture"} -->
+<html lang="en">
+<head>
+  <title>Capital cities — Claude</title>
+</head>
+<body>
+  <main>
+    <div data-testid="user-message">
+      <p>What is the capital of France?</p>
+    </div>
+    <div class="font-claude-response">
+      <div class="standard-markdown">
+        <p>The capital of France is Paris.</p>
+      </div>
+    </div>
+    <div data-testid="user-message">
+      <p>And Germany?</p>
+    </div>
+    <div class="font-claude-response">
+      <div class="standard-markdown">
+        <p>The capital of Germany is Berlin.</p>
+      </div>
+    </div>
+  </main>
+</body>
+</html>