diff --git a/src/cli.ts b/src/cli.ts index a40c7dd08..86df4f305 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -19,6 +19,7 @@ export interface ParseOptions { lang?: string; userAgent?: string; frontmatter?: boolean; + sourceUrl?: string; } interface ParseResult { @@ -61,7 +62,7 @@ export async function parseSource(source: string | undefined, options: ParseOpti }; let html: string; - let url: string | undefined; + let url: string | undefined = options.sourceUrl; const usesStdin = !source || source === '-'; const isUrl = !usesStdin && (source.startsWith('http://') || source.startsWith('https://')); @@ -72,6 +73,9 @@ export async function parseSource(source: string | undefined, options: ParseOpti } html = await readStdin(input); } else if (isUrl) { + // Positional URL is also the content URL; --source-url, if any, has been + // pre-seeded into `url` above and is overwritten here intentionally so the + // fetched URL is authoritative for a live fetch. url = source; const initialUA = options.userAgent || getInitialUA(source); html = await fetchPage(source, initialUA, options.lang); @@ -174,6 +178,7 @@ export function createProgram(): Command { .option('--debug', 'Enable debug mode') .option('-l, --lang ', 'Preferred language (BCP 47, e.g. en, fr, ja)') .option('-u, --user-agent ', 'Custom User-Agent header for HTTP requests (helps with 403/FORBIDDEN responses)') + .option('--source-url ', 'URL the input HTML originated from (enables site-specific extractors when source is stdin or a local file)') .action(async (source: string | undefined, options: ParseOptions) => { try { const { output } = await parseSource(source, options); diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 4522602c7..dba31a886 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -105,4 +105,49 @@ describe('CLI parseSource', () => { // commander camelCases --user-agent → options.userAgent, which parseSource reads. expect(option?.attributeName()).toBe('userAgent'); }); + + test('registers the --source-url flag', () => { + const parseCommand = createProgram().commands.find((c) => c.name() === 'parse'); + const option = parseCommand?.options.find((o) => o.long === '--source-url'); + + expect(option).toBeDefined(); + // commander camelCases --source-url → options.sourceUrl, which parseSource reads. + expect(option?.attributeName()).toBe('sourceUrl'); + }); + + test('uses --source-url to activate site-specific extractors for stdin input', async () => { + // The Claude conversation fixture has data-testid="user-message" and + // .font-claude-response divs, which the ClaudeExtractor recognises and + // formats with **You** / **Claude** author labels — but only when a + // claude.ai URL is supplied as the document URL. Without a URL, the + // extractor registry has no domain match, generic scoring runs, and + // the author labels are absent from the output. + const claudeFixture = readFileSync( + join(__dirname, 'fixtures', 'extractor--claude-conversation.html'), + 'utf-8' + ); + + // Baseline: without --source-url, ClaudeExtractor does not run, so the + // "**You**" / "**Claude**" author labels (only the conversation + // extractor emits these) are missing from the output. + const baseline = await parseSource( + '-', + { markdown: true }, + createMockStdin(claudeFixture) + ); + expect(baseline.output).not.toContain('**You**'); + expect(baseline.output).not.toContain('**Claude**'); + + // With --source-url pointing at claude.ai, ClaudeExtractor activates + // and emits the structured conversation turns with author labels. + const result = await parseSource( + '-', + { markdown: true, sourceUrl: 'https://claude.ai/chat/example-fixture' }, + createMockStdin(claudeFixture) + ); + expect(result.output).toContain('**You**'); + expect(result.output).toContain('**Claude**'); + expect(result.output).toContain('Paris'); + expect(result.output).toContain('Berlin'); + }); }); diff --git a/tests/expected/extractor--claude-conversation.md b/tests/expected/extractor--claude-conversation.md new file mode 100644 index 000000000..f94a80146 --- /dev/null +++ b/tests/expected/extractor--claude-conversation.md @@ -0,0 +1,30 @@ +```json +{ + "title": "Capital cities — Claude", + "author": "", + "site": "Claude", + "published": "" +} +``` + +**You** + +What is the capital of France? + +--- + +**Claude** + +The capital of France is Paris. + +--- + +**You** + +And Germany? + +--- + +**Claude** + +The capital of Germany is Berlin. \ No newline at end of file diff --git a/tests/fixtures/extractor--claude-conversation.html b/tests/fixtures/extractor--claude-conversation.html new file mode 100644 index 000000000..92dc1d4c8 --- /dev/null +++ b/tests/fixtures/extractor--claude-conversation.html @@ -0,0 +1,27 @@ + + + + + Capital cities — Claude + + +
+
+

What is the capital of France?

+
+
+
+

The capital of France is Paris.

+
+
+
+

And Germany?

+
+
+
+

The capital of Germany is Berlin.

+
+
+
+ +