From b7ff0e10529d56c986865d1e286fa622dd959670 Mon Sep 17 00:00:00 2001 From: George - David Apostolidis Date: Thu, 18 Jun 2026 11:11:07 +0200 Subject: [PATCH 1/3] Added Ece, Chemical, Mineral, Product and Systems Design engineering, needs more --- data/feeds.json | 7 +- scrapers/scraper_service.py | 5 +- scrapers/uowm_scraper.py | 174 ++++++++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 scrapers/uowm_scraper.py diff --git a/data/feeds.json b/data/feeds.json index e9955b2..c71be29 100644 --- a/data/feeds.json +++ b/data/feeds.json @@ -8,5 +8,10 @@ "name": "University of Macedonia", "type": "scraper", "scraper": "uom_news" + }, + { + "name": "University of Western Macedonia", + "type": "scraper", + "scraper": "uowm_news" } -] \ No newline at end of file +] diff --git a/scrapers/scraper_service.py b/scrapers/scraper_service.py index b071fe9..39cbd51 100644 --- a/scrapers/scraper_service.py +++ b/scrapers/scraper_service.py @@ -1,8 +1,9 @@ from scrapers.uom_scraper import UomNewsScraper - +from scrapers.uowm_scraper import UowmNewsScraper SCRAPER_REGISTRY = { "uom_news": UomNewsScraper, + "uowm_news": UowmNewsScraper, } @@ -13,4 +14,4 @@ def run_scraper(scraper_id: str) -> list[dict]: raise ValueError(f"Unknown scraper: {scraper_id}") scraper = scraper_class() - return scraper.scrape() \ No newline at end of file + return scraper.scrape() diff --git a/scrapers/uowm_scraper.py b/scrapers/uowm_scraper.py new file mode 100644 index 0000000..f79b361 --- /dev/null +++ b/scrapers/uowm_scraper.py @@ -0,0 +1,174 @@ +from datetime import datetime, timezone +from urllib.parse import urljoin + +import requests +from bs4 import BeautifulSoup + +from scrapers.base_scraper import BaseScraper + + +class UowmNewsScraper(BaseScraper): + name = "University of Western Macedonia" + + def __init__(self, max_pages_per_source: int = 5): + self.max_pages_per_source = max_pages_per_source + + self.sources = [ + { + "label": "UoWM Main News", + "base_url": "https://www.uowm.gr", + "first_page": "https://www.uowm.gr/en/news/", + "item_selector": ".post-list-item", + "title_selector": "h4.post-title a", + "date_selector": ".post-date span", + "summary_selector": ".post-item-excerpt", + }, + { + "label": "UoWM Electrical and Computer Engineering", + "base_url": "https://ece.uowm.gr", + "first_page": ( + "https://ece.uowm.gr/" + "%ce%b3%ce%b5%ce%bd%ce%b9%ce%ba%ce%b5%cf%83-" + "%ce%b1%ce%bd%ce%b1%ce%ba%ce%bf%ce%b9%ce%bd%cf%89%cf%83%ce%b5%ce%b9%cf%83/" + ), + "item_selector": ".rt-list-item", + "title_selector": "h3.entry-title a.tpg-post-link", + "date_selector": ".post-meta-tags .date a", + "summary_selector": ".tpg-excerpt-inner", + }, + { + "label": "UoWM Chemical Engineering", + "base_url": "https://chemeng.uowm.gr", + "first_page": "https://chemeng.uowm.gr/enimerosi/", + "item_selector": "article.post", + "title_selector": ".entry-title a, .cmsms_post_title a", + "date_selector": ".published, abbr.published, .cmsms_post_date", + "summary_selector": ".entry-content, .cmsms_post_content", + }, + { + "label": "UoWM Mineral Resources Engineering", + "base_url": "https://mre.uowm.gr", + "first_page": "https://mre.uowm.gr/en/category/announcements/", + "item_selector": ".post, article", + "title_selector": ".entry-title a, h2 a, h3 a", + "date_selector": ".entry-date, .posted-on, time", + "summary_selector": ".entry-summary, .entry-content", + }, + { + "label": "UoWM Product and Systems Design Engineering - First Years", + "base_url": "https://ide.uowm.gr", + "first_page": "https://ide.uowm.gr/category/protoeteis_25/", + "item_selector": "article.post", + "title_selector": "h2.entry-title a, .entry-title a", + "date_selector": ".entry-date, time, .posted-on", + "summary_selector": ".entry-summary, .entry-content", + }, + { + "label": "UoWM Product and Systems Design Engineering - News", + "base_url": "https://ide.uowm.gr", + "first_page": "https://ide.uowm.gr/en/category/news/", + "item_selector": "article.post", + "title_selector": "h2.entry-title a, .entry-title a", + "date_selector": ".entry-date, time, .posted-on", + "summary_selector": ".entry-summary, .entry-content", + }, + ] + + def scrape(self) -> list[dict]: + articles = [] + seen_links = set() + + for source in self.sources: + for page_number in range(1, self.max_pages_per_source + 1): + page_url = self.build_page_url(source["first_page"], page_number) + + try: + html = self.fetch_page(page_url) + page_articles = self.parse_page(html, source) + + if not page_articles: + break + + for article in page_articles: + link = article.get("link") + + if link and link not in seen_links: + seen_links.add(link) + articles.append(article) + + except Exception as error: + print( + f"Could not scrape {source['label']} " + f"page {page_number}: {error}" + ) + break + + return articles + + def build_page_url(self, first_page_url: str, page_number: int) -> str: + if page_number == 1: + return first_page_url + + return f"{first_page_url}page/{page_number}/" + + def fetch_page(self, url: str) -> str: + response = requests.get( + url, + timeout=15, + headers={ + "User-Agent": ( + "UniNews/0.1 (+https://github.com/open-source-uom/UniNews)" + ) + }, + ) + + response.raise_for_status() + return response.text + + def parse_page(self, html: str, source: dict) -> list[dict]: + soup = BeautifulSoup(html, "html.parser") + results = [] + + items = soup.select(source["item_selector"]) + + for item in items: + title_link = item.select_one(source["title_selector"]) + + if not title_link: + continue + + title = title_link.get_text(" ", strip=True) + link = urljoin(source["base_url"], title_link.get("href", "")) + + date_element = item.select_one(source["date_selector"]) + published = ( + date_element.get_text(" ", strip=True) + if date_element + else "Unknown date" + ) + + summary_element = item.select_one(source["summary_selector"]) + summary = ( + summary_element.get_text(" ", strip=True) if summary_element else "" + ) + + image_element = item.select_one("img") + image_url = "" + + if image_element and image_element.get("src"): + image_url = urljoin(source["base_url"], image_element.get("src")) + + results.append( + { + "university": self.name, + "source": source["label"], + "title": title, + "summary": summary, + "link": link, + "published": published, + "fetched_at": datetime.now(timezone.utc).isoformat(), + "image_url": image_url, + } + ) + + return results From c200bb6d80ad1080073c3060d92a448cf642365d Mon Sep 17 00:00:00 2001 From: George - David Apostolidis Date: Thu, 18 Jun 2026 18:17:46 +0200 Subject: [PATCH 2/3] Added most of the UoWM departments --- scrapers/uowm_scraper.py | 220 ++++++++++++++++++++++++++++++++------- 1 file changed, 182 insertions(+), 38 deletions(-) diff --git a/scrapers/uowm_scraper.py b/scrapers/uowm_scraper.py index f79b361..0fce261 100644 --- a/scrapers/uowm_scraper.py +++ b/scrapers/uowm_scraper.py @@ -10,7 +10,7 @@ class UowmNewsScraper(BaseScraper): name = "University of Western Macedonia" - def __init__(self, max_pages_per_source: int = 5): + def __init__(self, max_pages_per_source: int = 3): self.max_pages_per_source = max_pages_per_source self.sources = [ @@ -19,8 +19,8 @@ def __init__(self, max_pages_per_source: int = 5): "base_url": "https://www.uowm.gr", "first_page": "https://www.uowm.gr/en/news/", "item_selector": ".post-list-item", - "title_selector": "h4.post-title a", - "date_selector": ".post-date span", + "title_selector": "h4.post-title a, .post-title a", + "date_selector": ".post-date span, .post-date", "summary_selector": ".post-item-excerpt", }, { @@ -32,44 +32,128 @@ def __init__(self, max_pages_per_source: int = 5): "%ce%b1%ce%bd%ce%b1%ce%ba%ce%bf%ce%b9%ce%bd%cf%89%cf%83%ce%b5%ce%b9%cf%83/" ), "item_selector": ".rt-list-item", - "title_selector": "h3.entry-title a.tpg-post-link", - "date_selector": ".post-meta-tags .date a", + "title_selector": "h3.entry-title a.tpg-post-link, .entry-title a", + "date_selector": ".post-meta-tags .date a, .date", "summary_selector": ".tpg-excerpt-inner", }, { "label": "UoWM Chemical Engineering", "base_url": "https://chemeng.uowm.gr", "first_page": "https://chemeng.uowm.gr/enimerosi/", - "item_selector": "article.post", - "title_selector": ".entry-title a, .cmsms_post_title a", - "date_selector": ".published, abbr.published, .cmsms_post_date", - "summary_selector": ".entry-content, .cmsms_post_content", + "item_selector": "article.type-post, article.post", + "title_selector": "h3 a, h2 a, .entry-title a", + "date_selector": ".date, .entry-date, time, .posted-on", + "summary_selector": ".entry-summary, .entry-content, p", }, { "label": "UoWM Mineral Resources Engineering", "base_url": "https://mre.uowm.gr", "first_page": "https://mre.uowm.gr/en/category/announcements/", - "item_selector": ".post, article", - "title_selector": ".entry-title a, h2 a, h3 a", - "date_selector": ".entry-date, .posted-on, time", + "item_selector": ".post.type-post, .type-post", + "title_selector": "h2 a, .entry-title a", + "date_selector": ".entry-meta, .entry-date, time", "summary_selector": ".entry-summary, .entry-content", }, { "label": "UoWM Product and Systems Design Engineering - First Years", "base_url": "https://ide.uowm.gr", "first_page": "https://ide.uowm.gr/category/protoeteis_25/", - "item_selector": "article.post", - "title_selector": "h2.entry-title a, .entry-title a", - "date_selector": ".entry-date, time, .posted-on", + "item_selector": "article.type-post, article.post", + "title_selector": "h2 a, .entry-title a", + "date_selector": "time, .entry-date, .posted-on", "summary_selector": ".entry-summary, .entry-content", }, { "label": "UoWM Product and Systems Design Engineering - News", "base_url": "https://ide.uowm.gr", "first_page": "https://ide.uowm.gr/en/category/news/", - "item_selector": "article.post", - "title_selector": "h2.entry-title a, .entry-title a", - "date_selector": ".entry-date, time, .posted-on", + "item_selector": "article.type-post, article.post", + "title_selector": "h2 a, .entry-title a", + "date_selector": "time, .entry-date, .posted-on", + "summary_selector": ".entry-summary, .entry-content", + }, + { + "label": "UoWM Early Childhood Education", + "base_url": "https://nured.uowm.gr", + "first_page": "https://nured.uowm.gr/news/", + "item_selector": "article.type-post, article.post", + "title_selector": "h2 a, .entry-title a", + "date_selector": "time, .entry-date, .posted-on", + "summary_selector": ".entry-summary, .entry-content", + }, + { + "label": "UoWM Computer Science", + "base_url": "https://cs.uowm.gr", + "first_page": "https://cs.uowm.gr/en/home-page/", + "item_selector": ".latest-news-item", + "title_selector": ".latest-news-title a, h3 a", + "date_selector": ".latest-news-date, time, .posted-on", + "summary_selector": ".latest-news-text-content p, .latest-news-excerpt", + "max_pages": 1, + }, + { + "label": "UoWM Agriculture", + "base_url": "https://agro.uowm.gr", + "first_page": "https://agro.uowm.gr/home/", + "item_selector": ".uael-post-wrapper", + "title_selector": ".uael-post__title a, h5 a", + "date_selector": ".uael-post__date, .uael-post__meta-data, time", + "summary_selector": ".uael-post__excerpt, .uael-post__content-wrap", + "max_pages": 1, + }, + { + "label": "UoWM Regional and Cross-Border Development", + "base_url": "https://rdcbs.uowm.gr", + "first_page": "https://rdcbs.uowm.gr/nea/", + "item_selector": "article.post-list-item, .post-list-item", + "title_selector": "h3 a, .post-title a", + "date_selector": ".post-date, time", + "summary_selector": ".post-excerpt, .entry-summary, p", + }, + { + "label": "UoWM International and European Economic Studies", + "base_url": "https://iees.uowm.gr", + "first_page": "https://iees.uowm.gr/category/anakoinoseis/", + "item_selector": ".post.type-post, .type-post", + "title_selector": "h2 a, .entry-title a", + "date_selector": ".entry-meta, .entry-date, time", + "summary_selector": ".entry-summary, .entry-content", + }, + { + "label": "UoWM Business Administration", + "base_url": "https://ba.uowm.gr", + "first_page": "https://ba.uowm.gr/category/anakoinoseis/", + "item_selector": "article.type-post, article.post", + "title_selector": "h3 a, h2 a, .entry-title a", + "date_selector": ".post-date, .entry-date, time", + "summary_selector": ".entry-summary, .entry-content, p", + }, + { + "label": "UoWM Applied and Fine Arts", + "base_url": "https://eetf.uowm.gr", + "first_page": "https://eetf.uowm.gr/en/category/news/", + "item_selector": ".news-mini-wrap.type-post, .news-mini-wrap", + "title_selector": "h1 a, h2 a, .entry-title a", + "date_selector": "time, .news-meta", + "summary_selector": ".news-summary, .entry-summary, .entry-content", + }, + { + "label": "UoWM Occupational Therapy", + "base_url": "https://ot.uowm.gr", + "first_page": "https://ot.uowm.gr/", + "item_selector": "#left-sidebar .widget_text h3", + "title_selector": "a", + "date_selector": "", + "summary_selector": "", + "max_pages": 1, + }, + { + "label": "UoWM Midwifery", + "base_url": "https://mw.uowm.gr", + "first_page": "https://mw.uowm.gr/category/genikes-anakoinoseis/", + "item_selector": "article.type-post, article.post", + "title_selector": "h2 a, .entry-title a", + "date_selector": "time, .entry-date, .posted-on", "summary_selector": ".entry-summary, .entry-content", }, ] @@ -79,7 +163,9 @@ def scrape(self) -> list[dict]: seen_links = set() for source in self.sources: - for page_number in range(1, self.max_pages_per_source + 1): + max_pages = int(source.get("max_pages", self.max_pages_per_source)) + + for page_number in range(1, max_pages + 1): page_url = self.build_page_url(source["first_page"], page_number) try: @@ -109,7 +195,7 @@ def build_page_url(self, first_page_url: str, page_number: int) -> str: if page_number == 1: return first_page_url - return f"{first_page_url}page/{page_number}/" + return f"{first_page_url.rstrip('/')}/page/{page_number}/" def fetch_page(self, url: str) -> str: response = requests.get( @@ -123,6 +209,7 @@ def fetch_page(self, url: str) -> str: ) response.raise_for_status() + response.encoding = response.apparent_encoding return response.text def parse_page(self, html: str, source: dict) -> list[dict]: @@ -132,31 +219,25 @@ def parse_page(self, html: str, source: dict) -> list[dict]: items = soup.select(source["item_selector"]) for item in items: - title_link = item.select_one(source["title_selector"]) + title_link = self.find_title_link(item, source["title_selector"]) if not title_link: continue - title = title_link.get_text(" ", strip=True) - link = urljoin(source["base_url"], title_link.get("href", "")) + title = self.clean_text(title_link.get_text(" ", strip=True)) + href = title_link.get("href", "") - date_element = item.select_one(source["date_selector"]) - published = ( - date_element.get_text(" ", strip=True) - if date_element - else "Unknown date" - ) + if not title or not href or href.startswith("#"): + continue - summary_element = item.select_one(source["summary_selector"]) - summary = ( - summary_element.get_text(" ", strip=True) if summary_element else "" - ) + if title.startswith("http://") or title.startswith("https://"): + continue - image_element = item.select_one("img") - image_url = "" + link = urljoin(source["base_url"], href) - if image_element and image_element.get("src"): - image_url = urljoin(source["base_url"], image_element.get("src")) + published = self.extract_text(item, source.get("date_selector", "")) + summary = self.extract_text(item, source.get("summary_selector", "")) + image_url = self.extract_image_url(item, source["base_url"]) results.append( { @@ -165,10 +246,73 @@ def parse_page(self, html: str, source: dict) -> list[dict]: "title": title, "summary": summary, "link": link, - "published": published, + "published": published or "Unknown date", "fetched_at": datetime.now(timezone.utc).isoformat(), "image_url": image_url, } ) return results + + def find_title_link(self, item, selector: str): + for link in item.select(selector): + if link.name != "a": + link = link.select_one("a") + + if not link: + continue + + title = self.clean_text(link.get_text(" ", strip=True)) + href = link.get("href", "") + + if title and href and not href.startswith("#"): + return link + + return None + + def extract_text(self, item, selector: str) -> str: + if not selector: + return "" + + element = item.select_one(selector) + + if not element: + return "" + + return self.clean_text(element.get_text(" ", strip=True)) + + def extract_image_url(self, item, base_url: str) -> str: + image = item.select_one("img") + + if not image: + return "" + + src = ( + image.get("data-src") + or image.get("data-lazy-src") + or image.get("src") + or "" + ) + + if not src and image.get("srcset"): + src = image.get("srcset", "").split(",")[0].strip().split(" ")[0] + + if not src or src.startswith("data:image"): + return "" + + return urljoin(base_url, src) + + def clean_text(self, text: str) -> str: + cleaned = " ".join(text.split()) + + remove_phrases = [ + "Continue reading →", + "Read more", + "Περισσότερα", + "Συνεχίστε την ανάγνωση →", + ] + + for phrase in remove_phrases: + cleaned = cleaned.replace(phrase, "") + + return cleaned.strip() From 129040ea8042ed90a54fb6a2d9b9f360eba5a84c Mon Sep 17 00:00:00 2001 From: George - David Apostolidis Date: Wed, 1 Jul 2026 11:33:50 +0200 Subject: [PATCH 3/3] Seperated JSON sources into specific file --- data/university_lists/uowm_sources.json | 144 ++++++++++++++++++++++ feed_config.py | 47 +++++++ scrapers/university_source_loader.py | 44 +++++++ scrapers/uowm_scraper.py | 155 ++---------------------- 4 files changed, 243 insertions(+), 147 deletions(-) create mode 100644 data/university_lists/uowm_sources.json create mode 100644 feed_config.py create mode 100644 scrapers/university_source_loader.py diff --git a/data/university_lists/uowm_sources.json b/data/university_lists/uowm_sources.json new file mode 100644 index 0000000..dddae2e --- /dev/null +++ b/data/university_lists/uowm_sources.json @@ -0,0 +1,144 @@ +{ + "university": "University of Western Macedonia", + "default_max_pages": 3, + "sources": [ + { + "label": "UoWM Main News", + "base_url": "https://www.uowm.gr", + "first_page": "https://www.uowm.gr/en/news/", + "item_selector": ".post-list-item", + "title_selector": "h4.post-title a, .post-title a", + "date_selector": ".post-date span, .post-date", + "summary_selector": ".post-item-excerpt" + }, + { + "label": "UoWM Electrical and Computer Engineering", + "base_url": "https://ece.uowm.gr", + "first_page": "https://ece.uowm.gr/%ce%b3%ce%b5%ce%bd%ce%b9%ce%ba%ce%b5%cf%83-%ce%b1%ce%bd%ce%b1%ce%ba%ce%bf%ce%b9%ce%bd%cf%89%cf%83%ce%b5%ce%b9%cf%83/", + "item_selector": ".rt-list-item", + "title_selector": "h3.entry-title a.tpg-post-link, .entry-title a", + "date_selector": ".post-meta-tags .date a, .date", + "summary_selector": ".tpg-excerpt-inner" + }, + { + "label": "UoWM Chemical Engineering", + "base_url": "https://chemeng.uowm.gr", + "first_page": "https://chemeng.uowm.gr/enimerosi/", + "item_selector": "article.type-post, article.post", + "title_selector": "h3 a, h2 a, .entry-title a", + "date_selector": ".date, .entry-date, time, .posted-on", + "summary_selector": ".entry-summary, .entry-content, p" + }, + { + "label": "UoWM Mineral Resources Engineering", + "base_url": "https://mre.uowm.gr", + "first_page": "https://mre.uowm.gr/en/category/announcements/", + "item_selector": ".post.type-post, .type-post", + "title_selector": "h2 a, .entry-title a", + "date_selector": ".entry-meta, .entry-date, time", + "summary_selector": ".entry-summary, .entry-content" + }, + { + "label": "UoWM Product and Systems Design Engineering - First Years", + "base_url": "https://ide.uowm.gr", + "first_page": "https://ide.uowm.gr/category/protoeteis_25/", + "item_selector": "article.type-post, article.post", + "title_selector": "h2 a, .entry-title a", + "date_selector": "time, .entry-date, .posted-on", + "summary_selector": ".entry-summary, .entry-content" + }, + { + "label": "UoWM Product and Systems Design Engineering - News", + "base_url": "https://ide.uowm.gr", + "first_page": "https://ide.uowm.gr/en/category/news/", + "item_selector": "article.type-post, article.post", + "title_selector": "h2 a, .entry-title a", + "date_selector": "time, .entry-date, .posted-on", + "summary_selector": ".entry-summary, .entry-content" + }, + { + "label": "UoWM Early Childhood Education", + "base_url": "https://nured.uowm.gr", + "first_page": "https://nured.uowm.gr/news/", + "item_selector": "article.type-post, article.post", + "title_selector": "h2 a, .entry-title a", + "date_selector": "time, .entry-date, .posted-on", + "summary_selector": ".entry-summary, .entry-content" + }, + { + "label": "UoWM Computer Science", + "base_url": "https://cs.uowm.gr", + "first_page": "https://cs.uowm.gr/en/home-page/", + "item_selector": ".latest-news-item", + "title_selector": ".latest-news-title a, h3 a", + "date_selector": ".latest-news-date, time, .posted-on", + "summary_selector": ".latest-news-text-content p, .latest-news-excerpt", + "max_pages": 1 + }, + { + "label": "UoWM Agriculture", + "base_url": "https://agro.uowm.gr", + "first_page": "https://agro.uowm.gr/home/", + "item_selector": ".uael-post-wrapper", + "title_selector": ".uael-post__title a, h5 a", + "date_selector": ".uael-post__date, .uael-post__meta-data, time", + "summary_selector": ".uael-post__excerpt, .uael-post__content-wrap", + "max_pages": 1 + }, + { + "label": "UoWM Regional and Cross-Border Development", + "base_url": "https://rdcbs.uowm.gr", + "first_page": "https://rdcbs.uowm.gr/nea/", + "item_selector": "article.post-list-item, .post-list-item", + "title_selector": "h3 a, .post-title a", + "date_selector": ".post-date, time", + "summary_selector": ".post-excerpt, .entry-summary, p" + }, + { + "label": "UoWM International and European Economic Studies", + "base_url": "https://iees.uowm.gr", + "first_page": "https://iees.uowm.gr/category/anakoinoseis/", + "item_selector": ".post.type-post, .type-post", + "title_selector": "h2 a, .entry-title a", + "date_selector": ".entry-meta, .entry-date, time", + "summary_selector": ".entry-summary, .entry-content" + }, + { + "label": "UoWM Business Administration", + "base_url": "https://ba.uowm.gr", + "first_page": "https://ba.uowm.gr/category/anakoinoseis/", + "item_selector": "article.type-post, article.post", + "title_selector": "h3 a, h2 a, .entry-title a", + "date_selector": ".post-date, .entry-date, time", + "summary_selector": ".entry-summary, .entry-content, p" + }, + { + "label": "UoWM Applied and Fine Arts", + "base_url": "https://eetf.uowm.gr", + "first_page": "https://eetf.uowm.gr/en/category/news/", + "item_selector": ".news-mini-wrap.type-post, .news-mini-wrap", + "title_selector": "h1 a, h2 a, .entry-title a", + "date_selector": "time, .news-meta", + "summary_selector": ".news-summary, .entry-summary, .entry-content" + }, + { + "label": "UoWM Occupational Therapy", + "base_url": "https://ot.uowm.gr", + "first_page": "https://ot.uowm.gr/", + "item_selector": "#left-sidebar .widget_text h3", + "title_selector": "a", + "date_selector": "", + "summary_selector": "", + "max_pages": 1 + }, + { + "label": "UoWM Midwifery", + "base_url": "https://mw.uowm.gr", + "first_page": "https://mw.uowm.gr/category/genikes-anakoinoseis/", + "item_selector": "article.type-post, article.post", + "title_selector": "h2 a, .entry-title a", + "date_selector": "time, .entry-date, .posted-on", + "summary_selector": ".entry-summary, .entry-content" + } + ] +} diff --git a/feed_config.py b/feed_config.py new file mode 100644 index 0000000..4345dd4 --- /dev/null +++ b/feed_config.py @@ -0,0 +1,47 @@ +import json + +from settings import FEEDS_FILE + + +def load_builtin_feeds() -> list[dict]: + if not FEEDS_FILE.exists(): + return [] + + with open(FEEDS_FILE, "r", encoding="utf-8") as file: + return json.load(file) + + +def get_all_feeds(app_settings: dict) -> list[dict]: + builtin_feeds = load_builtin_feeds() + custom_feeds = app_settings.get("custom_rss_feeds", []) + + feeds = [] + seen = set() + + for feed in builtin_feeds + custom_feeds: + source_type = feed.get("type", "rss") + + if source_type == "rss": + name = feed.get("name", "").strip() + url = feed.get("url", "").strip() + + if not name or not url: + continue + + key = ("rss", url.lower()) + else: + name = feed.get("name", "").strip() + scraper = feed.get("scraper", "").strip() + + if not name or not scraper: + continue + + key = ("scraper", scraper.lower()) + + if key in seen: + continue + + seen.add(key) + feeds.append(feed) + + return feeds diff --git a/scrapers/university_source_loader.py b/scrapers/university_source_loader.py new file mode 100644 index 0000000..362a294 --- /dev/null +++ b/scrapers/university_source_loader.py @@ -0,0 +1,44 @@ +import json +from pathlib import Path + +from settings import DATA_DIR + +UNIVERSITY_LISTS_DIR = DATA_DIR / "university_lists" + + +def load_university_source_config(file_name: str) -> dict: + config_path = UNIVERSITY_LISTS_DIR / file_name + + if not config_path.exists(): + raise FileNotFoundError(f"Missing university source list: {config_path}") + + with open(config_path, "r", encoding="utf-8") as file: + config = json.load(file) + + validate_university_source_config(config, config_path) + + return config + + +def validate_university_source_config(config: dict, config_path: Path) -> None: + if "university" not in config: + raise ValueError(f"{config_path} is missing 'university'") + + if "sources" not in config: + raise ValueError(f"{config_path} is missing 'sources'") + + if not isinstance(config["sources"], list): + raise ValueError(f"{config_path}: 'sources' must be a list") + + required_fields = [ + "label", + "base_url", + "first_page", + "item_selector", + "title_selector", + ] + + for index, source in enumerate(config["sources"], start=1): + for field in required_fields: + if not source.get(field): + raise ValueError(f"{config_path}: source #{index} is missing '{field}'") diff --git a/scrapers/uowm_scraper.py b/scrapers/uowm_scraper.py index 0fce261..87a0136 100644 --- a/scrapers/uowm_scraper.py +++ b/scrapers/uowm_scraper.py @@ -5,158 +5,18 @@ from bs4 import BeautifulSoup from scrapers.base_scraper import BaseScraper +from scrapers.university_source_loader import load_university_source_config class UowmNewsScraper(BaseScraper): name = "University of Western Macedonia" - def __init__(self, max_pages_per_source: int = 3): - self.max_pages_per_source = max_pages_per_source - - self.sources = [ - { - "label": "UoWM Main News", - "base_url": "https://www.uowm.gr", - "first_page": "https://www.uowm.gr/en/news/", - "item_selector": ".post-list-item", - "title_selector": "h4.post-title a, .post-title a", - "date_selector": ".post-date span, .post-date", - "summary_selector": ".post-item-excerpt", - }, - { - "label": "UoWM Electrical and Computer Engineering", - "base_url": "https://ece.uowm.gr", - "first_page": ( - "https://ece.uowm.gr/" - "%ce%b3%ce%b5%ce%bd%ce%b9%ce%ba%ce%b5%cf%83-" - "%ce%b1%ce%bd%ce%b1%ce%ba%ce%bf%ce%b9%ce%bd%cf%89%cf%83%ce%b5%ce%b9%cf%83/" - ), - "item_selector": ".rt-list-item", - "title_selector": "h3.entry-title a.tpg-post-link, .entry-title a", - "date_selector": ".post-meta-tags .date a, .date", - "summary_selector": ".tpg-excerpt-inner", - }, - { - "label": "UoWM Chemical Engineering", - "base_url": "https://chemeng.uowm.gr", - "first_page": "https://chemeng.uowm.gr/enimerosi/", - "item_selector": "article.type-post, article.post", - "title_selector": "h3 a, h2 a, .entry-title a", - "date_selector": ".date, .entry-date, time, .posted-on", - "summary_selector": ".entry-summary, .entry-content, p", - }, - { - "label": "UoWM Mineral Resources Engineering", - "base_url": "https://mre.uowm.gr", - "first_page": "https://mre.uowm.gr/en/category/announcements/", - "item_selector": ".post.type-post, .type-post", - "title_selector": "h2 a, .entry-title a", - "date_selector": ".entry-meta, .entry-date, time", - "summary_selector": ".entry-summary, .entry-content", - }, - { - "label": "UoWM Product and Systems Design Engineering - First Years", - "base_url": "https://ide.uowm.gr", - "first_page": "https://ide.uowm.gr/category/protoeteis_25/", - "item_selector": "article.type-post, article.post", - "title_selector": "h2 a, .entry-title a", - "date_selector": "time, .entry-date, .posted-on", - "summary_selector": ".entry-summary, .entry-content", - }, - { - "label": "UoWM Product and Systems Design Engineering - News", - "base_url": "https://ide.uowm.gr", - "first_page": "https://ide.uowm.gr/en/category/news/", - "item_selector": "article.type-post, article.post", - "title_selector": "h2 a, .entry-title a", - "date_selector": "time, .entry-date, .posted-on", - "summary_selector": ".entry-summary, .entry-content", - }, - { - "label": "UoWM Early Childhood Education", - "base_url": "https://nured.uowm.gr", - "first_page": "https://nured.uowm.gr/news/", - "item_selector": "article.type-post, article.post", - "title_selector": "h2 a, .entry-title a", - "date_selector": "time, .entry-date, .posted-on", - "summary_selector": ".entry-summary, .entry-content", - }, - { - "label": "UoWM Computer Science", - "base_url": "https://cs.uowm.gr", - "first_page": "https://cs.uowm.gr/en/home-page/", - "item_selector": ".latest-news-item", - "title_selector": ".latest-news-title a, h3 a", - "date_selector": ".latest-news-date, time, .posted-on", - "summary_selector": ".latest-news-text-content p, .latest-news-excerpt", - "max_pages": 1, - }, - { - "label": "UoWM Agriculture", - "base_url": "https://agro.uowm.gr", - "first_page": "https://agro.uowm.gr/home/", - "item_selector": ".uael-post-wrapper", - "title_selector": ".uael-post__title a, h5 a", - "date_selector": ".uael-post__date, .uael-post__meta-data, time", - "summary_selector": ".uael-post__excerpt, .uael-post__content-wrap", - "max_pages": 1, - }, - { - "label": "UoWM Regional and Cross-Border Development", - "base_url": "https://rdcbs.uowm.gr", - "first_page": "https://rdcbs.uowm.gr/nea/", - "item_selector": "article.post-list-item, .post-list-item", - "title_selector": "h3 a, .post-title a", - "date_selector": ".post-date, time", - "summary_selector": ".post-excerpt, .entry-summary, p", - }, - { - "label": "UoWM International and European Economic Studies", - "base_url": "https://iees.uowm.gr", - "first_page": "https://iees.uowm.gr/category/anakoinoseis/", - "item_selector": ".post.type-post, .type-post", - "title_selector": "h2 a, .entry-title a", - "date_selector": ".entry-meta, .entry-date, time", - "summary_selector": ".entry-summary, .entry-content", - }, - { - "label": "UoWM Business Administration", - "base_url": "https://ba.uowm.gr", - "first_page": "https://ba.uowm.gr/category/anakoinoseis/", - "item_selector": "article.type-post, article.post", - "title_selector": "h3 a, h2 a, .entry-title a", - "date_selector": ".post-date, .entry-date, time", - "summary_selector": ".entry-summary, .entry-content, p", - }, - { - "label": "UoWM Applied and Fine Arts", - "base_url": "https://eetf.uowm.gr", - "first_page": "https://eetf.uowm.gr/en/category/news/", - "item_selector": ".news-mini-wrap.type-post, .news-mini-wrap", - "title_selector": "h1 a, h2 a, .entry-title a", - "date_selector": "time, .news-meta", - "summary_selector": ".news-summary, .entry-summary, .entry-content", - }, - { - "label": "UoWM Occupational Therapy", - "base_url": "https://ot.uowm.gr", - "first_page": "https://ot.uowm.gr/", - "item_selector": "#left-sidebar .widget_text h3", - "title_selector": "a", - "date_selector": "", - "summary_selector": "", - "max_pages": 1, - }, - { - "label": "UoWM Midwifery", - "base_url": "https://mw.uowm.gr", - "first_page": "https://mw.uowm.gr/category/genikes-anakoinoseis/", - "item_selector": "article.type-post, article.post", - "title_selector": "h2 a, .entry-title a", - "date_selector": "time, .entry-date, .posted-on", - "summary_selector": ".entry-summary, .entry-content", - }, - ] + def __init__(self): + config = load_university_source_config("uowm_sources.json") + + self.name = config["university"] + self.max_pages_per_source = int(config.get("default_max_pages", 3)) + self.sources = config["sources"] def scrape(self) -> list[dict]: articles = [] @@ -210,6 +70,7 @@ def fetch_page(self, url: str) -> str: response.raise_for_status() response.encoding = response.apparent_encoding + return response.text def parse_page(self, html: str, source: dict) -> list[dict]: