diff --git a/backend/app/scrap-registrations/scrap_registrations.ts b/backend/app/scrap-registrations/scrap_registrations.ts index f35f4b35..972d6044 100644 --- a/backend/app/scrap-registrations/scrap_registrations.ts +++ b/backend/app/scrap-registrations/scrap_registrations.ts @@ -192,22 +192,68 @@ export async function scrapCourseNameGroupsUrls( .eq(1) .text() .trim(); - const groups = $("div#layout-c22").find("div.usos-ui").children("usos-frame"); - groups.each((_, element) => { - const title = $(element).find("h2"); - if ( - title.text().includes("(jeszcze nie rozpoczęty)") || - title.text().includes("(w trakcie)") - ) { - const linksToSubCourses = $(element).find("usos-link").children("a"); - linksToSubCourses.each((__, el) => { - const url = $(el).attr("href"); - if (url !== undefined) { - urls.push(url); + const cycle = $("div#layout-c22") + .find("div.usos-ui") + .children("usos-frame.zajecia") + .filter((_, element) => { + const title = $(element).find("h2"); + return ( + title.text().includes("(jeszcze nie rozpoczęty)") || + title.text().includes("(w trakcie)") + ); + }) + .first(); + if (cycle.length > 0) { + // const linksToSubCourses = $(cycle).find("usos-link").children("a"); + // linksToSubCourses.each((__, el) => { + // const url = $(el).attr("href"); + // if (url !== undefined) { + // urls.push(url); + // } + // }); + + // they broke the group display in the course view - workaround + const linkToCalendar = $(cycle) + .find("usos-timetable") + .closest("a") + .attr("href"); + + if (linkToCalendar !== undefined) { + const calendarResponse = await fetchData(linkToCalendar); + if (!calendarResponse.ok) { + throw new Error( + `Got response code ${calendarResponse.status} ${calendarResponse.statusText} while fetching course calendar`, + ); + } + + const $$ = cheerio.load(await calendarResponse.text()); + + const courseIds = new Set(); + const links = $$("usos-timetable span[slot=dialog-info] a"); + for (const link of links) { + const href = $$(link).attr("href"); + if (href === undefined) { + continue; } - }); + const courseId = new URL(href).searchParams.get("zaj_cyk_id"); + if (courseId === null) { + continue; + } + courseIds.add(courseId); + } + + const baseUrl = new URL(courseUrl); + baseUrl.search = ""; + baseUrl.searchParams.set( + "_action", + "katalog2/przedmioty/pokazGrupyZajec", + ); + for (const id of courseIds) { + baseUrl.searchParams.set("zaj_cyk_id", id); + urls.push(baseUrl.href); + } } - }); + } return { courseName, urls, courseCode }; } diff --git a/backend/commands/scraper.ts b/backend/commands/scraper.ts index 494bc938..3db9e7dc 100644 --- a/backend/commands/scraper.ts +++ b/backend/commands/scraper.ts @@ -503,8 +503,9 @@ export default class Scraper extends BaseCommand { this.scrapingSemaphore = new Semaphore(this.maxUsosRequests); this.dbSemaphore = new Semaphore(this.maxDbRequests); - await this.ui - .tasks({ verbose: true }) + const tasks = this.ui.tasks({ verbose: true }); + + await tasks .add("Scrape departments", async (task) => { await this.departmentScrapeTask(task); return "Done"; @@ -534,5 +535,7 @@ export default class Scraper extends BaseCommand { return "Done"; }) .run(); + + this.exitCode = tasks.getState() === "succeeded" ? 0 : 1; } }