diff --git a/src/api.nim b/src/api.nim index dfcf41366..6200fb3be 100644 --- a/src/api.nim +++ b/src/api.nim @@ -51,6 +51,12 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} let url = graphListMembers ? {"variables": $variables} result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after) +proc getGraphArticle*(id: string): Future[Article] {.async.} = + let + variables = %*{"twitterArticleId": id} + url = graphArticle ? {"variables": $variables} + result = parseGraphArticle(await fetch(url, Api.userRestId)) + proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let diff --git a/src/consts.nim b/src/consts.nim index bb4e1a3c5..14d7cf166 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -25,6 +25,7 @@ const graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List" graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug" graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers" + graphArticle* = graphql / "rJMGbcr9LTsjVycjUmcnEg/TwitterArticleByRestId" timelineParams* = { "include_profile_interstitial_type": "0", diff --git a/src/formatters.nim b/src/formatters.nim index 363091752..0ec71f9f2 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -26,6 +26,8 @@ let userPicRegex = re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$" extRegex = re"(\.[A-z]+)$" illegalXmlRegex = re"(*UTF8)[^\x09\x0A\x0D\x20-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}]" + hashtagRegex = re"\B#(\w*[A-Za-z]\w*)\b" + mentionRegex = re"\B@(\w{1,15})\b" proc getUrlPrefix*(cfg: Config): string = if cfg.useHttps: https & cfg.hostname @@ -72,6 +74,13 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string = if absolute.len > 0 and "href" in result: result = result.replace("href=\"/", &"href=\"{absolute}/") +proc replaceHashtagsAndMentions*(body: string): string = + result = body + result = result.replacef(hashtagRegex, a( + "#$1", href = "/search?q=%23$1")) + result = result.replacef(mentionRegex, a( + "@$1", href = "/$1")) + proc getM3u8Url*(content: string): string = var matches: array[1, string] if re.find(content, m3u8Regex, matches) != -1: @@ -121,14 +130,14 @@ proc getTime*(tweet: Tweet): string = proc getRfc822Time*(tweet: Tweet): string = tweet.time.format("ddd', 'dd MMM yyyy HH:mm:ss 'GMT'") -proc getShortTime*(tweet: Tweet): string = +proc getShortTime*(time: DateTime): string = let now = now() - let since = now - tweet.time + let since = now - time - if now.year != tweet.time.year: - result = tweet.time.format("d MMM yyyy") + if now.year != time.year: + result = time.format("d MMM yyyy") elif since.inDays >= 1: - result = tweet.time.format("MMM d") + result = time.format("MMM d") elif since.inHours >= 1: result = $since.inHours & "h" elif since.inMinutes >= 1: diff --git a/src/nitter.nim b/src/nitter.nim index 2e868a44b..349e38a7d 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, tokens import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, - unsupported, embed, resolver, router_utils] + unsupported, embed, notes, resolver, router_utils] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" @@ -48,6 +48,7 @@ createListRouter(cfg) createStatusRouter(cfg) createSearchRouter(cfg) createMediaRouter(cfg) +createNotesRouter(cfg) createEmbedRouter(cfg) createRssRouter(cfg) createDebugRouter(cfg) @@ -100,4 +101,5 @@ routes: extend status, "" extend media, "" extend embed, "" + extend notes, "" extend debug, "" diff --git a/src/parser.nim b/src/parser.nim index fa877f9f3..9cf951952 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -435,3 +435,71 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = result.replies.content.add thread elif entryId.startsWith("cursor-bottom"): result.replies.bottom = e{"content", "itemContent", "value"}.getStr + +proc parseGraphArticle*(js: JsonNode): Article = + if not js{"errors"}.isNull: + return + + let article = js{"data", "twitterArticle"} + let meta = article{"metadata"} + + result = Article( + title: article{"title"}.getStr, + coverImage: article{"cover_image", "media_info", "original_img_url"}.getStr, + user: meta{"authorResults", "result", "legacy"}.parseUser, + time: meta{"publishedAtMs"}.getStr.parseInt.div(1000).fromUnix.utc, + ) + + let + content = article{"data", "contentStateJson"}.getStr.parseJson + + for p in content{"blocks"}: + var paragraph = ArticleParagraph( + text: p{"text"}.getStr, + baseType: parseEnum[ArticleType](p{"type"}.getStr) + ) + for sr in p{"inlineStyleRanges"}: + paragraph.inlineStyleRanges.add ArticleStyleRange( + offset: sr{"offset"}.getInt, + length: sr{"length"}.getInt, + style: parseEnum[ArticleStyle](sr{"style"}.getStr) + ) + for er in p{"entityRanges"}: + paragraph.entityRanges.add ArticleEntityRange( + offset: er{"offset"}.getInt, + length: er{"length"}.getInt, + key: er{"key"}.getInt + ) + result.paragraphs.add paragraph + + # Note: This is a map but the indices are integers so it's fine. + for _, jEntity in content{"entityMap"}: + var entity = ArticleEntity( + entityType: parseEnum[ArticleEntityType] jEntity{"type"}.getStr, + ) + case entity.entityType + of ArticleEntityType.link: + entity.url = jEntity{"data", "url"}.getStr + of ArticleEntityType.media: + for jMedia in jEntity{"data", "mediaItems"}: + entity.mediaIds.add jMedia{"mediaId"}.getStr + of ArticleEntityType.tweet: + entity.tweetId = jEntity{"data", "tweetId"}.getStr + of ArticleEntityType.twemoji: + entity.twemoji = jEntity{"data", "url"}.getStr + else: discard + + result.entities.add entity + + for m in article{"media"}: + let mediaInfo = m{"media_info"} + var media = ArticleMedia( + mediaType: parseEnum[ArticleMediaType](mediaInfo{"__typename"}.getStr) + ) + case media.mediaType + of ArticleMediaType.image: + media.url = mediaInfo{"original_img_url"}.getStr + of ArticleMediaType.gif: + media.url = mediaInfo{"variants"}[0]{"url"}.getStr + else: discard + result.media[m{"media_id"}.getStr] = media diff --git a/src/routes/notes.nim b/src/routes/notes.nim new file mode 100644 index 000000000..4a41de870 --- /dev/null +++ b/src/routes/notes.nim @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import asyncdispatch, tables, asyncfutures, options +import jester, karax/vdom +import ".."/[types, api] +import ../views/[notes, tweet, general] +import router_utils + +export api, notes, vdom, tweet, general, router_utils + +proc createNotesRouter*(cfg: Config) = + router notes: + get "/i/notes/@id": + let article = await getGraphArticle(@"id") + + if article == nil: + resp Http404 + + var tweetFutures: seq[Future[Conversation]] + for e in article.entities: + if e.entityType == ArticleEntityType.tweet: + tweetFutures.add getTweet(e.tweetId) + + let convs = await tweetFutures.all + + var tweets = initTable[int64, Tweet]() + for c in convs: + if c != nil and c.tweet != nil: + tweets[c.tweet.id] = c.tweet + + let + path = getPath() + prefs = cookiePrefs() + note = renderNote(article, tweets, path, prefs) + resp renderMain(note, request, cfg, prefs, titleText=article.title) diff --git a/src/routes/unsupported.nim b/src/routes/unsupported.nim index 0c085d4d8..3939b5ad3 100644 --- a/src/routes/unsupported.nim +++ b/src/routes/unsupported.nim @@ -21,5 +21,5 @@ proc createUnsupportedRouter*(cfg: Config) = feature() get "/i/@i?/?@j?": - cond @"i" notin ["status", "lists" , "user"] + cond @"i" notin ["status", "lists" , "user", "notes"] feature() diff --git a/src/sass/index.scss b/src/sass/index.scss index 9e2e34717..1d9e4669a 100644 --- a/src/sass/index.scss +++ b/src/sass/index.scss @@ -7,6 +7,7 @@ @import 'inputs'; @import 'timeline'; @import 'search'; +@import 'note'; body { // colors diff --git a/src/sass/note.scss b/src/sass/note.scss new file mode 100644 index 000000000..0bc3c0cea --- /dev/null +++ b/src/sass/note.scss @@ -0,0 +1,60 @@ +.note { + width: 600px; + margin: 0 auto; + background-color: var(--bg_panel); + font-family: sans-serif; + + img.cover { + margin: 0; + width: 100%; + } + + article { + padding: 20px; + + &>h1 { + display: inherit; + font-size: 2.5rem; + margin: 30px 0; + } + + &>p, + li { + font-size: 18px; + } + + &>p { + line-height: 1.5em; + margin: 30px 0; + word-wrap: break-word; + white-space: break-spaces; + } + + &>span.image { + text-align: center; + width: 100%; + + img, + video { + max-width: 100%; + border-radius: 20px; + margin: 0 auto; + } + } + + img.twemoji { + width: 18px; + height: 18px; + } + + &>ul>li, + &>ol>li { + line-height: 2em; + } + + &>iframe { + width: 100%; + height: 400px; + } + } +} \ No newline at end of file diff --git a/src/types.nim b/src/types.nim index 6f742d19e..0658b9876 100644 --- a/src/types.nim +++ b/src/types.nim @@ -38,6 +38,7 @@ type protectedUser = 22 couldntAuth = 32 doesntExist = 34 + invalidPermission = 37 userNotFound = 50 suspended = 63 rateLimited = 88 @@ -120,6 +121,70 @@ type PhotoRail* = seq[GalleryPhoto] + Article* = ref object + title*: string + coverImage*: string + user*: User + time*: DateTime + paragraphs*: seq[ArticleParagraph] + entities*: seq[ArticleEntity] + media*: Table[string, ArticleMedia] + + ArticleParagraph* = object + text*: string + baseType*: ArticleType + inlineStyleRanges*: seq[ArticleStyleRange] + entityRanges*: seq[ArticleEntityRange] + + ArticleType* {.pure.} = enum + headerOne = "header-one" + headerTwo = "header-two" + headerThree = "header-three" + orderedListItem = "ordered-list-item" + unorderedListItem = "unordered-list-item" + unstyled = "unstyled" + atomic = "atomic" + unknown + + ArticleStyleRange* = object + offset*: int + length*: int + style*: ArticleStyle + + ArticleStyle* {.pure.} = enum + bold = "BOLD" + italic = "ITALIC" + strikethrough = "STRIKETHROUGH" + unknown + + ArticleEntityRange* = object + offset*: int + length*: int + key*: int + + ArticleEntity* = object + entityType*: ArticleEntityType + url*: string + mediaIds*: seq[string] + tweetId*: string + twemoji*: string + + ArticleEntityType* {.pure.} = enum + link = "LINK" + media = "MEDIA" + tweet = "TWEET" + twemoji = "TWEMOJI" + unknown + + ArticleMedia* = object + mediaType*: ArticleMediaType + url*: string + + ArticleMediaType* {.pure.} = enum + image = "ApiImage" + gif = "ApiGif" + unknown + Poll* = object options*: seq[string] values*: seq[int] diff --git a/src/utils.nim b/src/utils.nim index 9002bbf72..e15b5466e 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -15,6 +15,7 @@ const "pic.twitter.com", "twimg.com", "abs.twimg.com", + "abs-0.twimg.com", "pbs.twimg.com", "video.twimg.com" ] diff --git a/src/views/general.nim b/src/views/general.nim index 5e96d02eb..87d30f2cd 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=18") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=19") link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2") if theme.len > 0: diff --git a/src/views/notes.nim b/src/views/notes.nim new file mode 100644 index 000000000..9c883234b --- /dev/null +++ b/src/views/notes.nim @@ -0,0 +1,195 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import strutils, tables, unicode, bitops +import karax/[karaxdsl, vdom] +from jester import Request + +import renderutils, tweet +import ".."/[types, utils, formatters] + +const doctype = "\n" + +proc getSmallPic(url: string): string = + result = url + if "?" notin url and not url.endsWith("placeholder.png"): + result &= "?name=small" + result = getPicUrl(result) + +proc renderMiniAvatar(user: User; prefs: Prefs): VNode = + let url = getPicUrl(user.getUserPic("_mini")) + buildHtml(): + img(class=(prefs.getAvatarClass & " mini"), src=url) + +proc renderNoteParagraph(articleParagraph: ArticleParagraph; article: Article; tweets: Table[int64, Tweet]; path: string; prefs: Prefs): VNode = + if articleParagraph.baseType == ArticleType.atomic: + let er = articleParagraph.entityRanges[0] + let entity = article.entities[er.key] + + case entity.entityType + of ArticleEntityType.media: + for id in entity.mediaIds: + let media = article.media.getOrDefault(id) + if media.url == "": + discard + case media.mediaType: + of ArticleMediaType.image: + let image = buildHtml(span(class="image")): + img(src=media.url.getSmallPic, alt="") + result = image + of ArticleMediaType.gif: + let video = buildHtml(span(class="image")): + video(src=media.url.getVidUrl, controls="", autoplay="", loop="") + result = video + else: discard + of ArticleEntityType.tweet: + let tweet = tweets.getOrDefault(entity.tweetId.parseInt, nil) + if tweet == nil: discard + result = renderTweet(tweet, prefs, path, mainTweet=true) + else: discard + else: + let text = articleParagraph.text + + case articleParagraph.baseType + of ArticleType.headerOne: + result = h1.newVNode() + of ArticleType.headerTwo: + result = h2.newVNode() + of ArticleType.headerThree: + result = h3.newVNode() + of ArticleType.orderedListItem: + result = li.newVNode() + of ArticleType.unorderedListItem: + result = li.newVNode() + of ArticleType.atomic: + result = nil + else: + result = p.newVNode() + + proc flushPlainText(target: VNode; start: int; len: int): void = + if articleParagraph.inlineStyleRanges.len == 0: + target.add verbatim text.runeSubStr(start, len).replaceHashtagsAndMentions + else: + + proc flushInternal(start: int, len: int, style: int): void = + let content = text.runeSubStr(start, len).replaceHashtagsAndMentions + if style == 0: + target.add text content + else: + let container = span.newVNode() + container.add text content + var styleStr = "" + if style.testBit(0): + styleStr.add "font-weight:bold;" + if style.testBit(1): + styleStr.add "font-style:italic;" + if style.testBit(2): + styleStr.add "text-decoration:line-through;" + container.setAttr("style", styleStr) + target.add container + + var + lastStyle = 0 + lastStart = start + + for i in start..(start + len): + var style = 0 + for styleRange in articleParagraph.inlineStyleRanges: + let + styleStart = styleRange.offset + styleEnd = styleStart + styleRange.length + if styleStart <= i and styleEnd > i: + case styleRange.style: + of ArticleStyle.bold: + style.setBit(0) + of ArticleStyle.italic: + style.setBit(1) + of ArticleStyle.strikethrough: + style.setBit(2) + else: discard + + if style != lastStyle: + if i > lastStart: + flushInternal(lastStart, i - lastStart, lastStyle) + + lastStyle = style + lastStart = i + + if lastStart < len: + flushInternal(lastStart, len - lastStart, lastStyle) + + var last = 0 + for er in articleParagraph.entityRanges: + # prevent karax from inserting whitespaces to fix wrapping + result.add text "" + + if er.offset > last: + flushPlainText(result, last, er.offset - last) + + let entity = article.entities[er.key] + case entity.entityType + of ArticleEntityType.link: + let link = buildHtml(a(href=entity.url)): + text text.runeSubStr(er.offset, er.length) + result.add link + of ArticleEntityType.twemoji: + let url = entity.twemoji.getSmallPic + let emoji = buildHtml(img(class="twemoji", src=url, alt="")) + result.add emoji + else: discard + + last = er.offset + er.length + + # flush remaining text + if last < text.len: + flushPlainText(result, last, text.len - last) + +proc renderNote*(article: Article; tweets: Table[int64, Tweet]; path: string; prefs: Prefs): VNode = + let cover = getSmallPic(article.coverImage) + let author = article.user + + # build header + let main = buildHtml(article): + h1: text article.title + + tdiv(class="author"): + renderMiniAvatar(author, prefs) + linkUser(author, class="fullname") + linkUser(author, class="username") + text " ยท " + text article.time.getShortTime + + # add paragraphs + var listType = ArticleType.unknown + var list: VNode = nil + + proc flushList() = + if list != nil: + main.add list + list = nil + listType = ArticleType.unknown + + for paragraph in article.paragraphs: + let node = renderNoteParagraph(paragraph, article, tweets, path, prefs) + + let currentType = paragraph.baseType + if currentType in [ArticleType.orderedListItem, ArticleType.unorderedListItem]: + if currentType != listType: + flushList() + + case currentType: + of ArticleType.orderedListItem: + list = ol.newVNode() + of ArticleType.unorderedListItem: + list = ul.newVNode() + else: discard + listType = currentType + list.add node + else: + flushList() + main.add node + + flushList() + + buildHtml(tdiv(class="note")): + img(class="cover", src=(cover), alt="") + + main diff --git a/src/views/tweet.nim b/src/views/tweet.nim index ea94e28c3..e9103f52c 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -38,7 +38,7 @@ proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode = span(class="tweet-date"): a(href=getLink(tweet), title=tweet.getTime): - text tweet.getShortTime + text tweet.time.getShortTime proc renderAlbum(tweet: Tweet): VNode = let @@ -253,7 +253,7 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode = span(class="tweet-date"): a(href=getLink(quote), title=quote.getTime): - text quote.getShortTime + text quote.time.getShortTime if quote.reply.len > 0: renderReply(quote)