From 958dd733e66b8c88bbab07022fc01c8f7e693bba Mon Sep 17 00:00:00 2001 From: kaznak Date: Fri, 19 Jun 2026 08:44:58 +0900 Subject: [PATCH] CLI: add --source-url flag for site-specific extraction on stdin/file input The extractor registry uses the document's URL to select site-specific extractors (e.g. ClaudeExtractor for claude.ai/chat/*, ChatGPTExtractor for chatgpt.com/c/*). When the CLI reads HTML from stdin or a local file the URL is left undefined, so the registry's `new URL('').hostname` throws, no extractor is selected, and generic scoring runs instead. For claude.ai chat captures the generic path extracts no content and the CLI exits with "No content could be extracted". Add a `--source-url ` option that lets the caller declare the URL the HTML originated from. The positional URL form continues to override it for live fetches. This is useful when post-processing archived web pages (SingleFile, single-file-cli, ArchiveBox, Obsidian Web Clipper output, etc.) where the original URL is known but the HTML is no longer fetched live. Includes a minimal anonymized claude.ai conversation fixture and a CLI test that verifies the **You**/**Claude** author labels (only the ClaudeExtractor emits them) appear only when --source-url is supplied. --- src/cli.ts | 7 ++- tests/cli.test.ts | 45 +++++++++++++++++++ .../extractor--claude-conversation.md | 30 +++++++++++++ .../extractor--claude-conversation.html | 27 +++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 tests/expected/extractor--claude-conversation.md create mode 100644 tests/fixtures/extractor--claude-conversation.html diff --git a/src/cli.ts b/src/cli.ts index a40c7dd08..86df4f305 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -19,6 +19,7 @@ export interface ParseOptions { lang?: string; userAgent?: string; frontmatter?: boolean; + sourceUrl?: string; } interface ParseResult { @@ -61,7 +62,7 @@ export async function parseSource(source: string | undefined, options: ParseOpti }; let html: string; - let url: string | undefined; + let url: string | undefined = options.sourceUrl; const usesStdin = !source || source === '-'; const isUrl = !usesStdin && (source.startsWith('http://') || source.startsWith('https://')); @@ -72,6 +73,9 @@ export async function parseSource(source: string | undefined, options: ParseOpti } html = await readStdin(input); } else if (isUrl) { + // Positional URL is also the content URL; --source-url, if any, has been + // pre-seeded into `url` above and is overwritten here intentionally so the + // fetched URL is authoritative for a live fetch. url = source; const initialUA = options.userAgent || getInitialUA(source); html = await fetchPage(source, initialUA, options.lang); @@ -174,6 +178,7 @@ export function createProgram(): Command { .option('--debug', 'Enable debug mode') .option('-l, --lang ', 'Preferred language (BCP 47, e.g. en, fr, ja)') .option('-u, --user-agent ', 'Custom User-Agent header for HTTP requests (helps with 403/FORBIDDEN responses)') + .option('--source-url ', 'URL the input HTML originated from (enables site-specific extractors when source is stdin or a local file)') .action(async (source: string | undefined, options: ParseOptions) => { try { const { output } = await parseSource(source, options); diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 4522602c7..dba31a886 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -105,4 +105,49 @@ describe('CLI parseSource', () => { // commander camelCases --user-agent → options.userAgent, which parseSource reads. expect(option?.attributeName()).toBe('userAgent'); }); + + test('registers the --source-url flag', () => { + const parseCommand = createProgram().commands.find((c) => c.name() === 'parse'); + const option = parseCommand?.options.find((o) => o.long === '--source-url'); + + expect(option).toBeDefined(); + // commander camelCases --source-url → options.sourceUrl, which parseSource reads. + expect(option?.attributeName()).toBe('sourceUrl'); + }); + + test('uses --source-url to activate site-specific extractors for stdin input', async () => { + // The Claude conversation fixture has data-testid="user-message" and + // .font-claude-response divs, which the ClaudeExtractor recognises and + // formats with **You** / **Claude** author labels — but only when a + // claude.ai URL is supplied as the document URL. Without a URL, the + // extractor registry has no domain match, generic scoring runs, and + // the author labels are absent from the output. + const claudeFixture = readFileSync( + join(__dirname, 'fixtures', 'extractor--claude-conversation.html'), + 'utf-8' + ); + + // Baseline: without --source-url, ClaudeExtractor does not run, so the + // "**You**" / "**Claude**" author labels (only the conversation + // extractor emits these) are missing from the output. + const baseline = await parseSource( + '-', + { markdown: true }, + createMockStdin(claudeFixture) + ); + expect(baseline.output).not.toContain('**You**'); + expect(baseline.output).not.toContain('**Claude**'); + + // With --source-url pointing at claude.ai, ClaudeExtractor activates + // and emits the structured conversation turns with author labels. + const result = await parseSource( + '-', + { markdown: true, sourceUrl: 'https://claude.ai/chat/example-fixture' }, + createMockStdin(claudeFixture) + ); + expect(result.output).toContain('**You**'); + expect(result.output).toContain('**Claude**'); + expect(result.output).toContain('Paris'); + expect(result.output).toContain('Berlin'); + }); }); diff --git a/tests/expected/extractor--claude-conversation.md b/tests/expected/extractor--claude-conversation.md new file mode 100644 index 000000000..f94a80146 --- /dev/null +++ b/tests/expected/extractor--claude-conversation.md @@ -0,0 +1,30 @@ +```json +{ + "title": "Capital cities — Claude", + "author": "", + "site": "Claude", + "published": "" +} +``` + +**You** + +What is the capital of France? + +--- + +**Claude** + +The capital of France is Paris. + +--- + +**You** + +And Germany? + +--- + +**Claude** + +The capital of Germany is Berlin. \ No newline at end of file diff --git a/tests/fixtures/extractor--claude-conversation.html b/tests/fixtures/extractor--claude-conversation.html new file mode 100644 index 000000000..92dc1d4c8 --- /dev/null +++ b/tests/fixtures/extractor--claude-conversation.html @@ -0,0 +1,27 @@ + + + + + Capital cities — Claude + + +
+
+

What is the capital of France?

+
+
+
+

The capital of France is Paris.

+
+
+
+

And Germany?

+
+
+
+

The capital of Germany is Berlin.

+
+
+
+ +