diff --git a/pdf/__pycache__/report_builder.cpython-312.pyc b/pdf/__pycache__/report_builder.cpython-312.pyc
new file mode 100644
index 0000000..75abe9b
Binary files /dev/null and b/pdf/__pycache__/report_builder.cpython-312.pyc differ
diff --git a/pdf/report_builder.py b/pdf/report_builder.py
new file mode 100644
index 0000000..e888618
--- /dev/null
+++ b/pdf/report_builder.py
@@ -0,0 +1,644 @@
+import re
+from collections import Counter
+from datetime import datetime
+from html import escape
+from pathlib import Path
+
+from reportlab.lib import colors
+from reportlab.lib.enums import TA_CENTER, TA_JUSTIFY, TA_LEFT
+from reportlab.lib.pagesizes import A4
+from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
+from reportlab.lib.units import inch
+from reportlab.platypus import (
+ HRFlowable,
+ KeepTogether,
+ ListFlowable,
+ ListItem,
+ PageBreak,
+ Paragraph,
+ SimpleDocTemplate,
+ Spacer,
+ Table,
+ TableStyle,
+)
+
+
+STOPWORDS = {
+ "about",
+ "above",
+ "after",
+ "again",
+ "against",
+ "also",
+ "among",
+ "because",
+ "before",
+ "being",
+ "between",
+ "could",
+ "during",
+ "from",
+ "have",
+ "into",
+ "more",
+ "most",
+ "other",
+ "over",
+ "said",
+ "same",
+ "some",
+ "such",
+ "than",
+ "that",
+ "their",
+ "there",
+ "these",
+ "they",
+ "this",
+ "through",
+ "under",
+ "were",
+ "where",
+ "which",
+ "while",
+ "with",
+ "would",
+}
+
+
+CHAPTERS = [
+ "Executive Summary",
+ "Research Scope And Method",
+ "Background And Context",
+ "Current Developments",
+ "Source-Based Analysis",
+ "Conclusion",
+ "References",
+]
+
+
+def _safe_filename(value):
+ cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", value.strip())
+ cleaned = cleaned.strip("._")
+ return cleaned[:80] or "research_report"
+
+
+def _clean_text(value):
+ return " ".join(str(value or "").split())
+
+
+def _paragraph(text, style):
+ return Paragraph(escape(_clean_text(text)), style)
+
+
+def _markup_paragraph(markup, style):
+ return Paragraph(str(markup or ""), style)
+
+
+def _split_sentences(text):
+ text = _clean_text(text)
+ if not text:
+ return []
+ sentences = re.split(r"(?<=[.!?])\s+(?=[A-Z0-9])", text)
+ return [item.strip() for item in sentences if len(item.strip()) > 20]
+
+
+def _paragraphs_from_text(text, max_chars=950):
+ sentences = _split_sentences(text)
+ if not sentences and text:
+ return [_clean_text(text)]
+
+ paragraphs = []
+ current = []
+ current_len = 0
+ for sentence in sentences:
+ if current and current_len + len(sentence) > max_chars:
+ paragraphs.append(" ".join(current))
+ current = []
+ current_len = 0
+ current.append(sentence)
+ current_len += len(sentence)
+
+ if current:
+ paragraphs.append(" ".join(current))
+ return paragraphs
+
+
+def _word_counts(text):
+ words = re.findall(r"[A-Za-z][A-Za-z-]{3,}", text.lower())
+ return Counter(word for word in words if word not in STOPWORDS)
+
+
+def _top_keywords(text, limit=8):
+ return [word for word, _ in _word_counts(text).most_common(limit)]
+
+
+def _extractive_summary(text, topic="", sentence_limit=5):
+ sentences = _split_sentences(text)
+ if len(sentences) <= sentence_limit:
+ return sentences
+
+ keyword_counts = _word_counts(text)
+ topic_words = set(re.findall(r"[A-Za-z][A-Za-z-]{2,}", topic.lower()))
+ scored = []
+ for index, sentence in enumerate(sentences):
+ words = re.findall(r"[A-Za-z][A-Za-z-]{3,}", sentence.lower())
+ score = sum(keyword_counts.get(word, 0) for word in words)
+ score += sum(4 for word in words if word in topic_words)
+ score = score / max(len(words), 1)
+ score += max(0, 1.2 - index * 0.08)
+ scored.append((score, index, sentence))
+
+ selected = sorted(scored, reverse=True)[:sentence_limit]
+ return [sentence for _, _, sentence in sorted(selected, key=lambda item: item[1])]
+
+
+def _article_text(item):
+ return item.get("full_text") or item.get("summary") or ""
+
+
+def _build_styles():
+ styles = getSampleStyleSheet()
+
+ styles["Title"].fontName = "Times-Bold"
+ styles["Title"].fontSize = 30
+ styles["Title"].leading = 36
+ styles["Title"].alignment = TA_CENTER
+ styles["Title"].textColor = colors.HexColor("#111827")
+
+ styles["Heading1"].fontName = "Times-Bold"
+ styles["Heading1"].fontSize = 18
+ styles["Heading1"].leading = 23
+ styles["Heading1"].spaceBefore = 18
+ styles["Heading1"].spaceAfter = 8
+ styles["Heading1"].textColor = colors.HexColor("#111827")
+
+ styles["Heading2"].fontName = "Times-Bold"
+ styles["Heading2"].fontSize = 13
+ styles["Heading2"].leading = 17
+ styles["Heading2"].spaceBefore = 10
+ styles["Heading2"].spaceAfter = 5
+ styles["Heading2"].textColor = colors.HexColor("#1F2937")
+
+ styles["BodyText"].fontName = "Times-Roman"
+ styles["BodyText"].fontSize = 10.5
+ styles["BodyText"].leading = 15
+ styles["BodyText"].alignment = TA_JUSTIFY
+ styles["BodyText"].spaceAfter = 6
+
+ styles.add(
+ ParagraphStyle(
+ name="CoverSubtitle",
+ parent=styles["BodyText"],
+ fontName="Helvetica",
+ fontSize=12,
+ leading=17,
+ alignment=TA_CENTER,
+ textColor=colors.HexColor("#374151"),
+ )
+ )
+ styles.add(
+ ParagraphStyle(
+ name="Meta",
+ parent=styles["BodyText"],
+ fontName="Helvetica",
+ textColor=colors.HexColor("#4B5563"),
+ fontSize=8.5,
+ leading=11,
+ alignment=TA_LEFT,
+ )
+ )
+ styles.add(
+ ParagraphStyle(
+ name="MetaCenter",
+ parent=styles["Meta"],
+ alignment=TA_CENTER,
+ )
+ )
+ styles.add(
+ ParagraphStyle(
+ name="Lead",
+ parent=styles["BodyText"],
+ fontName="Times-Italic",
+ fontSize=11.5,
+ leading=16,
+ textColor=colors.HexColor("#1F2937"),
+ )
+ )
+ styles.add(
+ ParagraphStyle(
+ name="Source",
+ parent=styles["BodyText"],
+ fontName="Helvetica",
+ textColor=colors.HexColor("#1D4ED8"),
+ fontSize=8.5,
+ leading=11,
+ alignment=TA_LEFT,
+ )
+ )
+ styles.add(
+ ParagraphStyle(
+ name="Toc",
+ parent=styles["BodyText"],
+ fontName="Helvetica",
+ fontSize=10.5,
+ leading=15,
+ alignment=TA_LEFT,
+ )
+ )
+ styles.add(
+ ParagraphStyle(
+ name="ChapterNumber",
+ parent=styles["BodyText"],
+ fontName="Helvetica-Bold",
+ fontSize=9,
+ leading=11,
+ textColor=colors.HexColor("#6B7280"),
+ alignment=TA_LEFT,
+ )
+ )
+ return styles
+
+
+def _rule(width=1, color="#9CA3AF", space_before=8, space_after=12):
+ return HRFlowable(
+ width="100%",
+ thickness=width,
+ color=colors.HexColor(color),
+ spaceBefore=space_before,
+ spaceAfter=space_after,
+ )
+
+
+def _chapter(story, number, title, styles):
+ story.append(Spacer(1, 0.08 * inch))
+ story.append(_paragraph(f"Chapter {number}", styles["ChapterNumber"]))
+ story.append(_paragraph(title, styles["Heading1"]))
+ story.append(_rule(width=0.7, color="#D1D5DB", space_before=2, space_after=10))
+
+
+def _bullet_list(items, styles, bullet_type="bullet"):
+ cleaned = [item for item in items if item]
+ if not cleaned:
+ return []
+ return [
+ ListFlowable(
+ [ListItem(_paragraph(item, styles["BodyText"])) for item in cleaned],
+ bulletType=bullet_type,
+ leftIndent=18,
+ bulletFontName="Helvetica",
+ )
+ ]
+
+
+def _source_table(rows, styles):
+ table = Table(rows, colWidths=[1.55 * inch, 4.45 * inch])
+ table.setStyle(
+ TableStyle(
+ [
+ ("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#F3F4F6")),
+ ("TEXTCOLOR", (0, 0), (-1, 0), colors.HexColor("#111827")),
+ ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"),
+ ("FONTNAME", (0, 1), (0, -1), "Helvetica-Bold"),
+ ("FONTSIZE", (0, 0), (-1, -1), 8.5),
+ ("VALIGN", (0, 0), (-1, -1), "TOP"),
+ ("GRID", (0, 0), (-1, -1), 0.25, colors.HexColor("#D1D5DB")),
+ ("ROWBACKGROUNDS", (0, 1), (-1, -1), [colors.white, colors.HexColor("#F9FAFB")]),
+ ("LEFTPADDING", (0, 0), (-1, -1), 6),
+ ("RIGHTPADDING", (0, 0), (-1, -1), 6),
+ ("TOPPADDING", (0, 0), (-1, -1), 5),
+ ("BOTTOMPADDING", (0, 0), (-1, -1), 5),
+ ]
+ )
+ )
+ return table
+
+
+def _add_cover(story, topic, wiki, news, styles):
+ generated_at = datetime.now().strftime("%d %B %Y")
+ source_count = (1 if wiki and wiki.get("url") else 0) + len(news)
+
+ story.append(Spacer(1, 1.35 * inch))
+ story.append(_rule(width=2.2, color="#111827", space_before=0, space_after=18))
+ story.append(_paragraph(f"{topic} Research Report", styles["Title"]))
+ story.append(Spacer(1, 0.18 * inch))
+ story.append(
+ _paragraph(
+ "A structured, source-backed project report with background context, recent developments, analysis, and references.",
+ styles["CoverSubtitle"],
+ )
+ )
+ story.append(Spacer(1, 0.35 * inch))
+ story.append(_rule(width=0.8, color="#9CA3AF", space_before=4, space_after=24))
+
+ rows = [
+ [_paragraph("Prepared On", styles["Meta"]), _paragraph(generated_at, styles["Meta"])],
+ [_paragraph("Primary Topic", styles["Meta"]), _paragraph(topic, styles["Meta"])],
+ [_paragraph("Sources Used", styles["Meta"]), _paragraph(str(source_count), styles["Meta"])],
+ [_paragraph("Report Type", styles["Meta"]), _paragraph("Automated research brief for project use", styles["Meta"])],
+ ]
+ story.append(_source_table(rows, styles))
+ story.append(Spacer(1, 1.1 * inch))
+ story.append(
+ _paragraph(
+ "Note: This report is generated from publicly available web sources. Review references before using it for formal submission.",
+ styles["MetaCenter"],
+ )
+ )
+ story.append(PageBreak())
+
+
+def _add_contents(story, styles):
+ story.append(_paragraph("Contents", styles["Heading1"]))
+ story.append(_rule(width=0.7, color="#D1D5DB", space_before=2, space_after=8))
+ for index, chapter in enumerate(CHAPTERS, start=1):
+ story.append(_markup_paragraph(f"{index}. {escape(chapter)}", styles["Toc"]))
+ story.append(PageBreak())
+
+
+def _add_executive_summary(story, topic, wiki, news, source_errors, styles):
+ _chapter(story, 1, "Executive Summary", styles)
+
+ lead_source = wiki.get("summary") if wiki else ""
+ if lead_source:
+ for paragraph in _paragraphs_from_text(lead_source, max_chars=850)[:3]:
+ story.append(_paragraph(paragraph, styles["Lead"]))
+ else:
+ story.append(
+ _paragraph(
+ f"This report reviews available public information on {topic}. It combines reference material and recent news coverage into a structured project-ready format.",
+ styles["Lead"],
+ )
+ )
+
+ collected_text = " ".join(
+ [wiki.get("summary", "") if wiki else ""]
+ + [section.get("text", "") for section in (wiki.get("sections", []) if wiki else [])]
+ + [_article_text(item) for item in news]
+ )
+ keywords = _top_keywords(collected_text, limit=6)
+ summary_points = [
+ f"The report uses {len(wiki.get('sections', [])) if wiki else 0} background section(s) and {len(news)} recent news item(s).",
+ f"Important recurring terms found in the collected material include: {', '.join(keywords)}." if keywords else "",
+ "Detailed article text is included where publisher pages allowed extraction.",
+ f"{len(source_errors)} source issue(s) were recorded and listed in the method section." if source_errors else "",
+ ]
+ for flowable in _bullet_list(summary_points, styles):
+ story.append(flowable)
+
+
+def _add_method(story, wiki, news, source_errors, styles):
+ _chapter(story, 2, "Research Scope And Method", styles)
+ story.append(
+ _paragraph(
+ "The report is assembled from live web sources at generation time. Wikipedia is used for baseline background, while Google News RSS is used to identify recent coverage. For each news item, the generator attempts to fetch and extract readable paragraph text from the publisher page.",
+ styles["BodyText"],
+ )
+ )
+
+ rows = [
+ [_paragraph("Source Area", styles["Meta"]), _paragraph("Collection Result", styles["Meta"])],
+ [
+ _paragraph("Wikipedia", styles["Meta"]),
+ _paragraph(
+ wiki.get("url", "No matching page returned.") if wiki else "No matching page returned.",
+ styles["Source"],
+ ),
+ ],
+ [
+ _paragraph("News", styles["Meta"]),
+ _paragraph(f"{len(news)} item(s) collected from Google News RSS.", styles["Meta"]),
+ ],
+ [
+ _paragraph("Article Text", styles["Meta"]),
+ _paragraph(
+ f"{sum(1 for item in news if item.get('full_text'))} item(s) contained extractable publisher article text.",
+ styles["Meta"],
+ ),
+ ],
+ ]
+ story.append(_source_table(rows, styles))
+
+ if source_errors:
+ story.append(_paragraph("Source Issues", styles["Heading2"]))
+ for flowable in _bullet_list(source_errors, styles):
+ story.append(flowable)
+
+
+def _add_background(story, wiki, styles):
+ _chapter(story, 3, "Background And Context", styles)
+ if not wiki:
+ story.append(_paragraph("No background content was available from Wikipedia for this topic.", styles["BodyText"]))
+ return
+
+ if wiki.get("url"):
+ story.append(_markup_paragraph(f"Reference source: {escape(wiki['url'])}", styles["Source"]))
+ story.append(Spacer(1, 0.08 * inch))
+
+ for section in wiki.get("sections", []):
+ story.append(_paragraph(section.get("title", "Background"), styles["Heading2"]))
+ for paragraph in _paragraphs_from_text(section.get("text", ""), max_chars=950):
+ story.append(_paragraph(paragraph, styles["BodyText"]))
+
+ child_items = []
+ for child in section.get("children", []):
+ child_text = child.get("text")
+ if child_text:
+ first_sentences = " ".join(_split_sentences(child_text)[:2])
+ child_items.append(f"{child.get('title')}: {first_sentences or child_text}")
+ for flowable in _bullet_list(child_items[:6], styles):
+ story.append(flowable)
+
+
+def _add_current_developments(story, topic, news, styles):
+ _chapter(story, 4, "Current Developments", styles)
+ if not news:
+ story.append(_paragraph("No recent news items were returned for this topic.", styles["BodyText"]))
+ return
+
+ for index, item in enumerate(news, start=1):
+ title = item.get("title") or "Untitled news item"
+ meta_parts = [part for part in [item.get("source"), item.get("published")] if part]
+ meta = " | ".join(meta_parts) or "Source metadata unavailable"
+ article_url = item.get("article_url") or item.get("link", "")
+ text = _article_text(item)
+ summary = _extractive_summary(text, topic=topic, sentence_limit=5)
+
+ block = [
+ _paragraph(f"{index}. {title}", styles["Heading2"]),
+ _paragraph(meta, styles["Meta"]),
+ ]
+ if item.get("content_status"):
+ block.append(_paragraph(item["content_status"], styles["Meta"]))
+ block.append(_markup_paragraph(f"Source: {escape(article_url)}", styles["Source"]))
+ story.append(KeepTogether(block))
+
+ if summary:
+ story.append(_paragraph("Detailed Summary", styles["Heading2"]))
+ for paragraph in _paragraphs_from_text(" ".join(summary), max_chars=850):
+ story.append(_paragraph(paragraph, styles["BodyText"]))
+
+ if item.get("full_text"):
+ key_points = _extractive_summary(item["full_text"], topic=topic, sentence_limit=4)
+ story.append(_paragraph("Key Points From The Article", styles["Heading2"]))
+ for flowable in _bullet_list(key_points, styles):
+ story.append(flowable)
+ elif item.get("summary"):
+ story.append(_paragraph("Available RSS Detail", styles["Heading2"]))
+ story.append(_paragraph(item["summary"], styles["BodyText"]))
+
+ story.append(Spacer(1, 0.1 * inch))
+
+
+def _add_analysis(story, topic, wiki, news, styles):
+ _chapter(story, 5, "Source-Based Analysis", styles)
+
+ background_text = " ".join(
+ [wiki.get("summary", "") if wiki else ""]
+ + [section.get("text", "") for section in (wiki.get("sections", []) if wiki else [])]
+ )
+ news_text = " ".join(_article_text(item) for item in news)
+ all_text = f"{background_text} {news_text}"
+
+ story.append(_paragraph("Main Themes", styles["Heading2"]))
+ keywords = _top_keywords(all_text, limit=10)
+ if keywords:
+ for flowable in _bullet_list([keyword.title() for keyword in keywords], styles):
+ story.append(flowable)
+ else:
+ story.append(_paragraph("No recurring themes could be extracted from the available text.", styles["BodyText"]))
+
+ story.append(_paragraph("Observations", styles["Heading2"]))
+ observations = []
+ if wiki:
+ observations.append(
+ f"The background source provides a reference foundation for {topic}, including {len(wiki.get('sections', []))} structured section(s)."
+ )
+ if news:
+ extracted = sum(1 for item in news if item.get("full_text"))
+ observations.append(
+ f"Recent coverage adds current context through {len(news)} news item(s), with detailed article text extracted for {extracted} item(s)."
+ )
+ if keywords:
+ observations.append(
+ f"The collected material repeatedly emphasizes {', '.join(keywords[:5])}, making these useful areas for project discussion."
+ )
+ observations.append(
+ "Use the references section to verify every important claim before final academic, business, or client submission."
+ )
+ for flowable in _bullet_list(observations, styles):
+ story.append(flowable)
+
+ story.append(_paragraph("Limitations", styles["Heading2"]))
+ limitations = [
+ "Some publisher sites block automated article extraction, so the report may use RSS summaries for those items.",
+ "The report is extractive and source-backed; it does not replace expert review or original research.",
+ "News coverage changes over time, so regenerate the report when current information is important.",
+ ]
+ for flowable in _bullet_list(limitations, styles):
+ story.append(flowable)
+
+
+def _add_conclusion(story, topic, wiki, news, styles):
+ _chapter(story, 6, "Conclusion", styles)
+
+ conclusion_parts = []
+ if wiki and wiki.get("summary"):
+ conclusion_parts.extend(_extractive_summary(wiki["summary"], topic=topic, sentence_limit=2))
+ if news:
+ latest_titles = [item.get("title") for item in news[:3] if item.get("title")]
+ if latest_titles:
+ conclusion_parts.append(
+ f"Recent coverage reviewed for this report includes: {'; '.join(latest_titles)}."
+ )
+
+ if not conclusion_parts:
+ conclusion_parts.append(
+ f"The available material for {topic} was limited, but the report captures the source status and references for follow-up research."
+ )
+
+ for paragraph in conclusion_parts:
+ story.append(_paragraph(paragraph, styles["BodyText"]))
+
+ story.append(
+ _paragraph(
+ "For project use, the strongest next step is to convert the themes, background facts, and current developments into your own argument, supported by the references listed below.",
+ styles["Lead"],
+ )
+ )
+
+
+def _add_references(story, wiki, news, styles):
+ _chapter(story, 7, "References", styles)
+
+ items = []
+ if wiki and wiki.get("url"):
+ items.append(f"Wikipedia - {wiki.get('title', 'Topic page')}: {wiki['url']}")
+ for item in news:
+ title = item.get("title", "News item")
+ source = item.get("source", "Publisher")
+ published = item.get("published", "date unavailable")
+ url = item.get("article_url") or item.get("link", "")
+ items.append(f"{source} ({published}) - {title}: {url}")
+
+ if not items:
+ story.append(_paragraph("No external sources were available.", styles["BodyText"]))
+ return
+
+ story.append(
+ ListFlowable(
+ [ListItem(_paragraph(item, styles["Source"])) for item in items],
+ bulletType="1",
+ leftIndent=18,
+ bulletFontName="Helvetica",
+ )
+ )
+
+
+def _page_frame(canvas, doc):
+ canvas.saveState()
+ width, height = A4
+ canvas.setStrokeColor(colors.HexColor("#D1D5DB"))
+ canvas.setLineWidth(0.35)
+ canvas.line(0.62 * inch, height - 0.48 * inch, width - 0.62 * inch, height - 0.48 * inch)
+ canvas.line(0.62 * inch, 0.48 * inch, width - 0.62 * inch, 0.48 * inch)
+
+ canvas.setFont("Helvetica", 8)
+ canvas.setFillColor(colors.HexColor("#6B7280"))
+ canvas.drawString(0.65 * inch, height - 0.38 * inch, "Research Report")
+ canvas.drawRightString(width - 0.65 * inch, 0.31 * inch, f"Page {doc.page}")
+ canvas.restoreState()
+
+
+def build_report(topic, wiki, news, output_dir="reports", source_errors=None):
+ source_errors = source_errors or []
+ output_path = Path(output_dir)
+ output_path.mkdir(parents=True, exist_ok=True)
+ filename = f"{_safe_filename(topic)}_Report.pdf"
+ report_path = output_path / filename
+
+ doc = SimpleDocTemplate(
+ str(report_path),
+ pagesize=A4,
+ rightMargin=0.72 * inch,
+ leftMargin=0.72 * inch,
+ topMargin=0.76 * inch,
+ bottomMargin=0.72 * inch,
+ title=f"{topic} Research Report",
+ author="Research Report Generator",
+ )
+ styles = _build_styles()
+ story = []
+
+ _add_cover(story, topic, wiki, news, styles)
+ _add_contents(story, styles)
+ _add_executive_summary(story, topic, wiki, news, source_errors, styles)
+ _add_method(story, wiki, news, source_errors, styles)
+ _add_background(story, wiki, styles)
+ _add_current_developments(story, topic, news, styles)
+ _add_analysis(story, topic, wiki, news, styles)
+ _add_conclusion(story, topic, wiki, news, styles)
+ _add_references(story, wiki, news, styles)
+
+ doc.build(story, onFirstPage=_page_frame, onLaterPages=_page_frame)
+ return report_path.resolve()
diff --git a/reports/Artificial_Intellgence_Report.pdf b/reports/Artificial_Intellgence_Report.pdf
new file mode 100644
index 0000000..a6ef1f6
Binary files /dev/null and b/reports/Artificial_Intellgence_Report.pdf differ
diff --git a/reports/Tata_Motors_Report.pdf b/reports/Tata_Motors_Report.pdf
new file mode 100644
index 0000000..a502f46
Binary files /dev/null and b/reports/Tata_Motors_Report.pdf differ
diff --git a/reports/tata_moters_Report.pdf b/reports/tata_moters_Report.pdf
new file mode 100644
index 0000000..57d4563
Binary files /dev/null and b/reports/tata_moters_Report.pdf differ
diff --git a/reports_test/Artificial_intelligence_Report.pdf b/reports_test/Artificial_intelligence_Report.pdf
new file mode 100644
index 0000000..8b7ba8a
Binary files /dev/null and b/reports_test/Artificial_intelligence_Report.pdf differ
diff --git a/sources/__pycache__/news_source.cpython-312.pyc b/sources/__pycache__/news_source.cpython-312.pyc
new file mode 100644
index 0000000..aef5bcb
Binary files /dev/null and b/sources/__pycache__/news_source.cpython-312.pyc differ
diff --git a/sources/__pycache__/wikipedia_source.cpython-312.pyc b/sources/__pycache__/wikipedia_source.cpython-312.pyc
new file mode 100644
index 0000000..d069d81
Binary files /dev/null and b/sources/__pycache__/wikipedia_source.cpython-312.pyc differ
diff --git a/sources/news_source.py b/sources/news_source.py
new file mode 100644
index 0000000..8249b60
--- /dev/null
+++ b/sources/news_source.py
@@ -0,0 +1,306 @@
+import json
+import re
+from datetime import datetime
+from html import unescape
+from urllib.parse import quote, quote_plus, urlparse
+
+import feedparser
+import requests
+from bs4 import BeautifulSoup
+
+
+class NewsSourceError(RuntimeError):
+ """Raised when the news source cannot be reached or parsed."""
+
+
+ARTICLE_USER_AGENT = (
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/125.0 Safari/537.36 ResearchReportGenerator/2.0"
+)
+MAX_ARTICLE_CHARS = 9000
+
+
+def _clean_text(value):
+ raw = str(value or "")
+ if "<" in raw or "&" in raw:
+ text = BeautifulSoup(raw, "html.parser").get_text(" ")
+ else:
+ text = raw
+ return " ".join(unescape(text).split())
+
+
+def _entry_date(entry):
+ parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
+ if not parsed:
+ return ""
+ return datetime(*parsed[:6]).strftime("%Y-%m-%d")
+
+
+def _google_news_article_id(url):
+ parsed = urlparse(url)
+ if "news.google.com" not in parsed.netloc or "/articles/" not in parsed.path:
+ return ""
+ return parsed.path.rsplit("/articles/", 1)[-1].strip("/")
+
+
+def _decode_google_news_url(url, timeout):
+ article_id = _google_news_article_id(url)
+ if not article_id:
+ return url
+
+ try:
+ session = requests.Session()
+ page = session.get(
+ f"https://news.google.com/articles/{article_id}",
+ timeout=timeout,
+ headers={"User-Agent": ARTICLE_USER_AGENT},
+ )
+ page.raise_for_status()
+ soup = BeautifulSoup(page.text, "html.parser")
+ decode_data = soup.find(attrs={"data-n-a-sg": True})
+ if not decode_data:
+ return url
+
+ timestamp = decode_data.get("data-n-a-ts")
+ signature = decode_data.get("data-n-a-sg")
+ if not timestamp or not signature:
+ return url
+
+ inner = (
+ '["garturlreq",[['
+ '"en-US","US",["FINANCE_TOP_INDICES","WEB_TEST_1_0_0"],null,null,1,1,'
+ '"US:en",null,180,null,null,null,null,null,0,null,null,[1608992183,723341000]],'
+ '"en-US","US",1,[2,3,4,8],1,0,"655000234",0,0,null,0],'
+ f'"{article_id}",{timestamp},"{signature}"]'
+ )
+ request_body = f'[[["Fbv4je","{inner.replace(chr(34), chr(92) + chr(34))}",null,"generic"]]]'
+ response = session.post(
+ "https://news.google.com/_/DotsSplashUi/data/batchexecute?rpcids=Fbv4je",
+ data=f"f.req={quote(request_body)}",
+ timeout=timeout,
+ headers={
+ "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
+ "User-Agent": ARTICLE_USER_AGENT,
+ },
+ )
+ response.raise_for_status()
+
+ for candidate in re.findall(r"https?://[^\\\"]+", response.text):
+ if "news.google.com" not in urlparse(candidate).netloc:
+ return candidate
+ except requests.RequestException:
+ return url
+
+ return url
+
+
+def _canonical_url(soup, fallback):
+ tag = soup.find("link", rel=lambda value: value and "canonical" in value)
+ if tag and tag.get("href"):
+ return tag["href"]
+ tag = soup.find("meta", property="og:url")
+ if tag and tag.get("content"):
+ return tag["content"]
+ return fallback
+
+
+def _page_title(soup):
+ for selector in [
+ ("meta", {"property": "og:title"}),
+ ("meta", {"name": "twitter:title"}),
+ ]:
+ tag = soup.find(*selector)
+ if tag and tag.get("content"):
+ return _clean_text(tag["content"])
+ if soup.title and soup.title.string:
+ return _clean_text(soup.title.string)
+ return ""
+
+
+def _is_useful_paragraph(text):
+ if len(text) < 70:
+ return False
+ lowered = text.lower()
+ blocked = [
+ "accept cookies",
+ "advertisement",
+ "all rights reserved",
+ "enable javascript",
+ "privacy policy",
+ "sign up",
+ "subscribe",
+ ]
+ return not any(item in lowered for item in blocked)
+
+
+def _extract_paragraphs(soup):
+ for tag in soup(["script", "style", "noscript", "nav", "footer", "header", "aside", "form"]):
+ tag.decompose()
+
+ containers = soup.find_all("article")
+ if not containers:
+ containers = soup.find_all("main")
+ if not containers:
+ containers = [soup.body or soup]
+
+ paragraphs = []
+ seen = set()
+ for container in containers:
+ for paragraph in container.find_all("p"):
+ text = _clean_text(paragraph.get_text(" "))
+ fingerprint = text.lower()
+ if fingerprint in seen or not _is_useful_paragraph(text):
+ continue
+ seen.add(fingerprint)
+ paragraphs.append(text)
+
+ if sum(len(item) for item in paragraphs) >= MAX_ARTICLE_CHARS:
+ return paragraphs
+
+ return paragraphs
+
+
+def _json_ld_nodes(value):
+ if isinstance(value, dict):
+ yield value
+ for key in ("@graph", "itemListElement"):
+ child = value.get(key)
+ if child:
+ yield from _json_ld_nodes(child)
+ elif isinstance(value, list):
+ for item in value:
+ yield from _json_ld_nodes(item)
+
+
+def _extract_structured_text(soup):
+ candidates = []
+ for script in soup.find_all("script", attrs={"type": "application/ld+json"}):
+ raw = script.string or script.get_text()
+ if not raw:
+ continue
+ try:
+ data = json.loads(raw)
+ except json.JSONDecodeError:
+ continue
+
+ for node in _json_ld_nodes(data):
+ body = node.get("articleBody") or node.get("description")
+ if isinstance(body, str) and _is_useful_paragraph(_clean_text(body)):
+ candidates.append(_clean_text(body))
+
+ for attrs in [
+ {"property": "og:description"},
+ {"name": "description"},
+ {"name": "twitter:description"},
+ ]:
+ tag = soup.find("meta", attrs=attrs)
+ if tag and tag.get("content"):
+ text = _clean_text(tag["content"])
+ if _is_useful_paragraph(text):
+ candidates.append(text)
+
+ seen = set()
+ unique = []
+ for text in candidates:
+ fingerprint = text.lower()
+ if fingerprint not in seen:
+ seen.add(fingerprint)
+ unique.append(text)
+ return "\n\n".join(unique)
+
+
+def _fetch_article_content(url, timeout):
+ resolved_url = _decode_google_news_url(url, timeout=timeout)
+ empty_result = {
+ "article_url": resolved_url,
+ "article_title": "",
+ "full_text": "",
+ "content_status": "Article text was not available from the publisher page.",
+ }
+
+ try:
+ response = requests.get(
+ resolved_url,
+ timeout=timeout,
+ allow_redirects=True,
+ headers={"User-Agent": ARTICLE_USER_AGENT},
+ )
+ response.raise_for_status()
+ except requests.RequestException:
+ return {
+ **empty_result,
+ "content_status": "Article text could not be fetched from the publisher page.",
+ }
+
+ content_type = response.headers.get("content-type", "").lower()
+ if "html" not in content_type and "text" not in content_type:
+ return empty_result
+
+ soup = BeautifulSoup(response.text, "html.parser")
+ paragraphs = _extract_paragraphs(soup)
+ full_text = "\n\n".join(paragraphs) or _extract_structured_text(soup)
+ if len(full_text) > MAX_ARTICLE_CHARS:
+ full_text = full_text[:MAX_ARTICLE_CHARS].rsplit(" ", 1)[0]
+
+ return {
+ "article_url": _canonical_url(soup, response.url),
+ "article_title": _page_title(soup),
+ "full_text": full_text,
+ "content_status": "Article text extracted." if full_text else empty_result["content_status"],
+ }
+
+
+def get_news(topic, limit=8, timeout=15, include_article_content=True, article_timeout=12):
+ """Return recent news results for a topic from Google News RSS."""
+ if not topic or not topic.strip():
+ raise ValueError("Topic is required to fetch news.")
+
+ if limit <= 0:
+ return []
+
+ query = quote_plus(topic.strip())
+ url = f"https://news.google.com/rss/search?q={query}&hl=en-US&gl=US&ceid=US:en"
+ try:
+ response = requests.get(
+ url,
+ timeout=timeout,
+ headers={"User-Agent": "ResearchReportGenerator/2.0"},
+ )
+ response.raise_for_status()
+ except requests.RequestException as exc:
+ raise NewsSourceError("Unable to fetch the news feed. Please try again later.") from exc
+
+ feed = feedparser.parse(response.content)
+
+ if getattr(feed, "bozo", False):
+ raise NewsSourceError("Unable to parse the news feed. Please try again later.")
+
+ results = []
+ seen_links = set()
+ for entry in feed.entries:
+ link = _clean_text(getattr(entry, "link", ""))
+ if not link or link in seen_links:
+ continue
+
+ seen_links.add(link)
+ item = {
+ "title": _clean_text(getattr(entry, "title", "")) or "Untitled news item",
+ "link": link,
+ "published": _entry_date(entry),
+ "source": _clean_text(getattr(getattr(entry, "source", None), "title", "")),
+ "summary": _clean_text(getattr(entry, "summary", "")),
+ "article_url": link,
+ "article_title": "",
+ "full_text": "",
+ "content_status": "Article content fetch was disabled.",
+ }
+ if include_article_content:
+ item.update(_fetch_article_content(link, timeout=article_timeout))
+
+ results.append(item)
+
+ if len(results) >= limit:
+ break
+
+ return results
diff --git a/sources/wikipedia_source.py b/sources/wikipedia_source.py
new file mode 100644
index 0000000..9c31c25
--- /dev/null
+++ b/sources/wikipedia_source.py
@@ -0,0 +1,56 @@
+import wikipediaapi
+
+
+class WikipediaSourceError(RuntimeError):
+ """Raised when Wikipedia cannot be reached or parsed."""
+
+
+def _section_to_dict(section, depth=0, max_depth=1):
+ if depth > max_depth:
+ return None
+
+ text = " ".join((section.text or "").split())
+ children = [
+ child
+ for child in (_section_to_dict(item, depth + 1, max_depth) for item in section.sections)
+ if child
+ ]
+
+ if not text and not children:
+ return None
+
+ return {
+ "title": section.title,
+ "text": text,
+ "children": children,
+ }
+
+
+def get_wikipedia_content(topic, section_limit=8):
+ """Return summary and high-value sections from Wikipedia for a topic."""
+ if not topic or not topic.strip():
+ raise ValueError("Topic is required to fetch Wikipedia content.")
+
+ try:
+ wiki = wikipediaapi.Wikipedia(user_agent="ResearchReportGenerator/2.0", language="en")
+ page = wiki.page(topic.strip())
+
+ if not page.exists():
+ return {}
+
+ sections = []
+ for section in page.sections:
+ section_data = _section_to_dict(section)
+ if section_data:
+ sections.append(section_data)
+ if len(sections) >= section_limit:
+ break
+
+ return {
+ "title": page.title,
+ "summary": " ".join((page.summary or "").split()),
+ "url": page.fullurl,
+ "sections": sections,
+ }
+ except Exception as exc:
+ raise WikipediaSourceError("Unable to contact Wikipedia. Please try again later.") from exc