diff --git a/src/extractors/x-oembed.ts b/src/extractors/x-oembed.ts index 48bad5ee7..255936437 100644 --- a/src/extractors/x-oembed.ts +++ b/src/extractors/x-oembed.ts @@ -63,6 +63,13 @@ interface FxTwitterResponse { }; } +type FxTwitterTweet = FxTwitterResponse['tweet']; + +interface FxTwitterThreadResponse { + code: number; + thread: FxTwitterTweet[]; +} + interface FxTwitterArticleMediaEntity { media_id: string; media_info: { @@ -193,24 +200,44 @@ export class XOembedExtractor extends BaseExtractor { const match = this.url.match(/\/([a-zA-Z0-9_][a-zA-Z0-9_]{0,14})\/(status|article)\/(\d+)/); if (!match) return null; + // Try Thread (v2) first try { - const data = await this.fetchFxTwitter(match[1], match[3]); - // If it's an article, use the rich article renderer - if (data.tweet?.article) { - return this.buildArticleResult(data); + const data = await this.fetchFxTwitter(null, match[3]); + if ('thread' in data) { + // If it's an article, use the rich article renderer + if (data.thread[0]?.article) { + return this.buildArticleResult(data); + } + // Otherwise use the full tweet text from FxTwitter + if (data.thread[0]?.text) { + return this.buildThreadResult(data); + } } - // Otherwise use the full tweet text from FxTwitter - if (data.tweet?.text) { - return this.buildTweetResult(data); + } catch { /* fall through */ } + + // Fallback to v1 + try { + const data = await this.fetchFxTwitter(match[1], match[3]); + if ('tweet' in data) { + // If it's an article, use the rich article renderer + if (data.tweet?.article) { + return this.buildArticleResult(data); + } + // Otherwise use the full tweet text from FxTwitter + if (data.tweet?.text) { + return this.buildTweetResult(data); + } } return null; - } catch { - return null; - } + } catch { /* fall through */ } + + return null; } - private async fetchFxTwitter(username: string, id: string): Promise { - const apiUrl = `https://api.fxtwitter.com/${username}/status/${id}`; + private async fetchFxTwitter(username: string | null, id: string): Promise { + const apiUrl = username + ? `https://api.fxtwitter.com/${username}/status/${id}` + : `https://api.fxtwitter.com/2/thread/${id}`; const response = await this.fetch(apiUrl, { headers: { 'User-Agent': 'Mozilla/5.0 (compatible; Defuddle/1.0; +https://defuddle.md)', @@ -233,13 +260,13 @@ export class XOembedExtractor extends BaseExtractor { } } - private buildArticleResult(data: FxTwitterResponse): ExtractorResult { - const article = data.tweet.article!; + private buildArticleResult(data: FxTwitterResponse | FxTwitterThreadResponse): ExtractorResult { + const article = 'thread' in data ? data.thread[0]?.article! : data.tweet.article!; const { blocks, entityMap } = article.content; const mediaEntities = article.media_entities || []; const contentHtml = this.renderArticle(blocks, entityMap, article.cover_media, mediaEntities); - const handle = `@${data.tweet.author.screen_name}`; - const published = this.toDateString(article.created_at) ?? this.toDateString(data.tweet.created_at); + const handle = `@${'thread' in data ? data.thread[0]?.author.screen_name : data.tweet.author.screen_name}`; + const published = this.toDateString(article.created_at) ?? this.toDateString('thread' in data ? data.thread[0]?.created_at : data.tweet.created_at); return { content: contentHtml, @@ -275,6 +302,34 @@ export class XOembedExtractor extends BaseExtractor { }; } + private buildThreadResult(data: FxTwitterThreadResponse): ExtractorResult | null { + const thread = data.thread; + if (!thread || thread.length === 0) return null; + + const [original, ...threadTweets] = thread; + const originalTweet = this.renderTweet(original); + const threadContent = threadTweets + .map(tweet => this.renderTweet(tweet)) + .filter(Boolean); + const postContent = [originalTweet, ...threadContent].join('\n
\n'); + const handle = `@${original.author.screen_name}`; + const contentHtml = buildContentHtml('twitter', postContent, ''); + const published = this.toDateString(original.created_at); + const description = (original.text || '').trim().slice(0, 140).replace(/\s+/g, ' '); + + return { + content: contentHtml, + contentHtml, + variables: { + title: this.postTitle(handle, 'X'), + author: handle, + site: 'X (Twitter)', + description, + ...(published && { published }), + } + }; + } + /** * Convert a Unicode code-point index to a UTF-16 code-unit offset. * FxTwitter facet indices count code points (emoji = 1) but JavaScript diff --git a/tests/x-oembed-surrogates.test.ts b/tests/x-oembed-surrogates.test.ts index 8b01e3b10..1fcb9805f 100644 --- a/tests/x-oembed-surrogates.test.ts +++ b/tests/x-oembed-surrogates.test.ts @@ -146,3 +146,128 @@ describe('XOembedExtractor — FxTwitter surrogate-pair index adjustment', () => ); }); }); + +describe('XOembedExtractor — FxTwitter thread support and fallback', () => { + const THREAD_API_RESPONSE = { + code: 200, + thread: [ + { + url: 'https://x.com/testuser/status/123', + id: '123', + text: 'This is the first tweet of the thread.', + author: { + screen_name: 'testuser', + name: 'Test User', + }, + created_at: '2026-01-01T00:00:00.000Z', + }, + { + url: 'https://x.com/testuser/status/124', + id: '124', + text: 'This is the second tweet of the thread.', + author: { + screen_name: 'testuser', + name: 'Test User', + }, + created_at: '2026-01-01T00:01:00.000Z', + }, + ], + }; + + const SINGLE_TWEET_API_RESPONSE = { + code: 200, + tweet: { + url: 'https://x.com/testuser/status/123', + id: '123', + text: 'This is the first tweet of the thread.', + author: { + screen_name: 'testuser', + name: 'Test User', + }, + created_at: '2026-01-01T00:00:00.000Z', + }, + }; + + function mockFetchForThreadFallback(threadData: any, tweetData: any, shouldThreadFail: boolean) { + return async (input: RequestInfo | URL, _init?: RequestInit) => { + const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url; + if (url.includes('/2/thread/')) { + if (shouldThreadFail) { + return new Response('Rate Limited', { status: 429 }); + } + return new Response(JSON.stringify(threadData), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (url.includes('/status/')) { + return new Response(JSON.stringify(tweetData), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ html: '
oEmbed fallback
', author_name: 'testuser', author_url: 'https://x.com/testuser' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }; + } + + test('successfully fetches and renders full thread via v2 API', async () => { + const doc = parseLinkedomHTML( + 'Test', + 'https://x.com/testuser/status/123', + ); + + const result = await Defuddle(doc, 'https://x.com/testuser/status/123', { + fetch: mockFetchForThreadFallback(THREAD_API_RESPONSE, SINGLE_TWEET_API_RESPONSE, false), + }); + + expect(result.content).toContain('This is the first tweet of the thread.'); + expect(result.content).toContain('This is the second tweet of the thread.'); + expect(result.content).toContain('
'); + }); + + test('cascades to v1 single tweet when v2 rate limited (429)', async () => { + const doc = parseLinkedomHTML( + 'Test', + 'https://x.com/testuser/status/123', + ); + + const result = await Defuddle(doc, 'https://x.com/testuser/status/123', { + fetch: mockFetchForThreadFallback(THREAD_API_RESPONSE, SINGLE_TWEET_API_RESPONSE, true), + }); + + expect(result.content).toContain('This is the first tweet of the thread.'); + expect(result.content).not.toContain('This is the second tweet of the thread.'); + }); + + test('cascades to oEmbed when both v2 and v1 fail', async () => { + const doc = parseLinkedomHTML( + 'Test', + 'https://x.com/testuser/status/123', + ); + + const failingFetch = async (input: RequestInfo | URL, _init?: RequestInit) => { + const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url; + if (url.includes('api.fxtwitter.com')) { + return new Response('Fail', { status: 500 }); + } + return new Response(JSON.stringify({ + html: '

oEmbed fallback text

', + author_name: 'Test User', + author_url: 'https://twitter.com/testuser', + provider_name: 'Twitter' + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }; + + const result = await Defuddle(doc, 'https://x.com/testuser/status/123', { + fetch: failingFetch, + }); + + expect(result.content).toContain('oEmbed fallback text'); + }); +});