From 53cc9e0205a6fe478afed57bd1a4adb8f52ec3b0 Mon Sep 17 00:00:00 2001 From: Karim-Agami Date: Mon, 13 Apr 2026 03:29:33 +0200 Subject: [PATCH 1/6] feat: implement crawl session management with CRUD operations - Added new models and schemas for CrawlSession and related data. - Created controller and routes for managing crawl sessions. - Implemented service layer for business logic related to crawl sessions. - Introduced a queue system for handling crawl tasks asynchronously. - Added a worker to process crawl jobs and manage their lifecycle. - Created a fake crawler for testing purposes. - Updated package dependencies to use the latest version of @coveritlabs/contracts. - Modified .gitignore to exclude .npmrc file. --- .gitignore | 1 + .npmrc | 1 - package-lock.json | 48 ++-- package.json | 4 +- prisma/schema.prisma | 19 +- .../controllers/crawlSession.controller.ts | 110 ++++++++++ src/api/routes/crawlSession.routes.ts | 33 +++ src/models/crawlSession.ts | 65 ++++++ src/queues/crawl.queue.ts | 24 ++ src/services/crawlSession.service.ts | 205 ++++++++++++++++++ src/workers/crawler.worker.ts | 139 ++++++++++++ src/workers/fake_crawler.py | 44 ++++ 12 files changed, 664 insertions(+), 29 deletions(-) delete mode 100644 .npmrc create mode 100644 src/api/controllers/crawlSession.controller.ts create mode 100644 src/api/routes/crawlSession.routes.ts create mode 100644 src/models/crawlSession.ts create mode 100644 src/queues/crawl.queue.ts create mode 100644 src/services/crawlSession.service.ts create mode 100644 src/workers/crawler.worker.ts create mode 100644 src/workers/fake_crawler.py diff --git a/.gitignore b/.gitignore index 104d867..942629c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ coverage .env.* /src/generated/prisma +.npmrc \ No newline at end of file diff --git a/.npmrc b/.npmrc deleted file mode 100644 index 9e09b04..0000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -@coveritlabs:registry=https://npm.pkg.github.com diff --git a/package-lock.json b/package-lock.json index 64b154f..4cdb940 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@asteasolutions/zod-to-openapi": "^8.4.3", "@bufbuild/protobuf": "^2.11.0", - "@coveritlabs/contracts": "^1.3.1", + "@coveritlabs/contracts": "^1.6.1", "@prisma/adapter-pg": "^7.4.2", "@prisma/client": "^7.4.2", "argon2": "^0.44.0", @@ -150,7 +150,6 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -981,9 +980,9 @@ } }, "node_modules/@coveritlabs/contracts": { - "version": "1.3.1", - "resolved": "https://npm.pkg.github.com/download/@coveritlabs/contracts/1.3.1/0f70431fbaa063e1f253869e4cdfe5416fa0910a", - "integrity": "sha512-0bq7xGkR5Rsfsz7eXPibNY6hjlhfiqejPi3WTSnl/4/pUsLgNtQ1aoEfzqNYXzcKjAsyzspsOtzlwdw/DXxiRg==", + "version": "1.6.1", + "resolved": "https://npm.pkg.github.com/download/@coveritlabs/contracts/1.6.1/2a65eea060d869746a3d556a25e6f43262bc2cab", + "integrity": "sha512-1YP5O2ghicIsgeWMS7QGFjfc+aLxQWByZ7Nc0dQDnZbNb/GEFJQlnmUFXmWw7xM2PFArUFBUBzwxc2PIF1v0zA==", "dependencies": { "@bufbuild/protobuf": "^2.5.0" } @@ -1033,8 +1032,7 @@ "version": "0.3.15", "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.3.15.tgz", "integrity": "sha512-Cj++n1Mekf9ETfdc16TlDi+cDDQF0W7EcbyRHYOAeZdsAe8M/FJg18itDTSwyHfar2WIezawM9o0EKaRGVKygQ==", - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/@electric-sql/pglite-socket": { "version": "0.0.20", @@ -1348,6 +1346,7 @@ "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "chardet": "^2.1.1", "iconv-lite": "^0.7.0" @@ -1370,6 +1369,7 @@ "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, @@ -1387,6 +1387,7 @@ "integrity": "sha512-t2IEY+unGHOzAaVM5Xx6DEWKeXlDDcNPeDyUpsRc6CUhBfU3VQOEl+Vssh7VNp1dR8MdUJBWhuObjXCsVpjN5g==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -3266,7 +3267,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.15.tgz", "integrity": "sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg==", "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -3451,7 +3451,6 @@ "integrity": "sha512-XZzOmihLIr8AD1b9hL9ccNMzEMWt/dE2u7NyTY9jJG6YNiNthaD5XtUHVF2uCXZ15ng+z2hT3MVuxnUYhq6k1g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.57.0", "@typescript-eslint/types": "8.57.0", @@ -3984,7 +3983,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4514,7 +4512,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -4786,7 +4783,8 @@ "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/chevrotain": { "version": "10.5.0", @@ -4904,6 +4902,7 @@ "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", "dev": true, "license": "ISC", + "peer": true, "engines": { "node": ">= 12" } @@ -5065,7 +5064,6 @@ "integrity": "sha512-gwAPAVTy/j5YcOOebcCRIijn+mSjWJC+IYKivTu6aG8Ei/scoXgfsMRnuAk6b0GRste2J4NGxVdMN3ZpfNaVaw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "cachedir": "2.3.0", "cz-conventional-changelog": "3.3.0", @@ -5414,7 +5412,6 @@ "integrity": "sha512-hr4ihw+DBqcvrsEDioRO31Z17x71pUYoNe/4h6Z0wB72p7MU7/9gH8Q3s12NFhHPfYBBOV3qyfUxmr/Yn3shnQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -5508,7 +5505,8 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/cz-conventional-changelog": { "version": "3.3.0", @@ -5962,7 +5960,6 @@ "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -7249,7 +7246,6 @@ "resolved": "https://registry.npmjs.org/hono/-/hono-4.11.4.tgz", "integrity": "sha512-U7tt8JsyrxSRKspfhtLET79pU8K+tInj5QZXs1jSugO1Vq5dFj3kmZsRldo29mTBfcjDRVRXrEZ6LS63Cog9ZA==", "license": "MIT", - "peer": true, "engines": { "node": ">=16.9.0" } @@ -7793,7 +7789,6 @@ "integrity": "sha512-AkXIIFcaazymvey2i/+F94XRnM6TsVLZDhBMLsd1Sf/W0wzsvvpjeyUrCZD6HGG4SDYPgDJDBKeiJTBb10WzMg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/core": "30.3.0", "@jest/types": "30.3.0", @@ -10294,6 +10289,7 @@ "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", "dev": true, "license": "ISC", + "peer": true, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } @@ -10982,7 +10978,6 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", "license": "MIT", - "peer": true, "dependencies": { "pg-connection-string": "^2.12.0", "pg-pool": "^3.13.0", @@ -11401,7 +11396,6 @@ "integrity": "sha512-n30qZpWehaYQzigLjmuPisyEsvOzHt7bZeRyg8gZ5DvJo9FGjD+gNaY59Ns3hlLD5/jZH5GBeftIss0jDbUoLg==", "hasInstallScript": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@prisma/config": "7.5.0", "@prisma/dev": "0.20.0", @@ -11843,6 +11837,7 @@ "integrity": "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=0.12.0" } @@ -11920,7 +11915,8 @@ "version": "0.27.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/secure-json-parse": { "version": "4.1.0", @@ -12755,7 +12751,6 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -12912,7 +12907,6 @@ "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13208,6 +13202,7 @@ "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -13278,6 +13273,7 @@ "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -13294,6 +13290,7 @@ "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "color-name": "~1.1.4" }, @@ -13306,7 +13303,8 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/wrappy": { "version": "1.0.2", @@ -13440,6 +13438,7 @@ "integrity": "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -13492,7 +13491,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 914de73..31f9e78 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "dependencies": { "@asteasolutions/zod-to-openapi": "^8.4.3", "@bufbuild/protobuf": "^2.11.0", - "@coveritlabs/contracts": "^1.3.1", + "@coveritlabs/contracts": "^1.6.1", "@prisma/adapter-pg": "^7.4.2", "@prisma/client": "^7.4.2", "argon2": "^0.44.0", @@ -80,4 +80,4 @@ "overrides": { "ioredis": "^5.10.0" } -} +} \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index dbbe3e1..e8280f0 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -105,7 +105,7 @@ model TargetApplicationVersion { updatedAt DateTime @updatedAt @map("updated_at") targetApplication TargetApplication @relation(fields: [targetApplicationId], references: [id], onDelete: Cascade) - + crawlSessions CrawlSession[] @@unique([targetApplicationId, version]) @@map("target_application_versions") } @@ -126,4 +126,21 @@ model RegressionCodebase { targetApplication TargetApplication @relation(fields: [targetApplicationId], references: [id], onDelete: Cascade) @@map("regression_codebases") @@unique([targetApplicationId, repositoryUrl]) +} + +model CrawlSession { + id String @id @default(uuid()) @map("crawl_session_id") @db.Uuid + appVersionId String @map("app_version_id") @db.Uuid + status String @default("NEW") + triggerType String @map("trigger_type") + config Json + stateCount Int @default(0) @map("state_count") + transitionCount Int @default(0) @map("transition_count") + createdAt DateTime @default(now()) @map("created_at") + startedAt DateTime? @map("started_at") + finishedAt DateTime? @map("finished_at") + error String? @db.Text + appVersion TargetApplicationVersion @relation(fields: [appVersionId], references: [id], onDelete: Cascade) + + @@map("crawl_sessions") } \ No newline at end of file diff --git a/src/api/controllers/crawlSession.controller.ts b/src/api/controllers/crawlSession.controller.ts new file mode 100644 index 0000000..7a15c87 --- /dev/null +++ b/src/api/controllers/crawlSession.controller.ts @@ -0,0 +1,110 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + +import { Request, Response } from 'express'; +import * as crawlService from '@services/crawlSession.service'; +import { StatusCodes } from 'http-status-codes'; +import { ZodError } from 'zod'; +import { + AppVersionParamsSchema, + CrawlSessionParamsSchema, + CreateCrawlSessionRequestSchema, + GetSessionsQuerySchema, +} from '@models/crawlSession'; + +function handleControllerError(res: Response, error: unknown): void { + if (error instanceof ZodError) { + res.status(StatusCodes.BAD_REQUEST).json({ + message: 'Validation failed', + details: error.issues, + }); + return; + } + if (typeof error === 'object' && error !== null && 'code' in error && error.code === 'P2025') { + res.status(StatusCodes.NOT_FOUND).json({ message: 'Resource not found' }); + return; + } + if (error instanceof Error) { + res.status(StatusCodes.CONFLICT).json({ message: error.message }); + return; + } + + console.error('[Crawl Controller Error]:', error); + res.status(StatusCodes.INTERNAL_SERVER_ERROR).json({ + message: 'An unexpected internal server error occurred', + }); +} + + +export const getSessions = async (req: Request, res: Response) => { + try { + const { app_version_id } = AppVersionParamsSchema.parse(req.params); + const query = GetSessionsQuerySchema.parse(req.query); + const result = await crawlService.getSessions(app_version_id, query); + res.json(result); + } catch (e) { + handleControllerError(res, e); + } +}; + +export const createSession = async (req: Request, res: Response) => { + try { + const { app_version_id } = AppVersionParamsSchema.parse(req.params); + const body = CreateCrawlSessionRequestSchema.parse(req.body); + const result = await crawlService.createSession(app_version_id, body.triggerType, body.crawlConfig); + res.status(StatusCodes.CREATED).json(result); + } catch (e) { + handleControllerError(res, e); + } +}; + +export const getSessionDetails = async (req: Request, res: Response) => { + try { + const { crawl_session_id } = CrawlSessionParamsSchema.parse(req.params); + const result = await crawlService.getSessionDetails(crawl_session_id); + res.json(result); + } catch (e) { + handleControllerError(res, e); + } +}; + +export const deleteSession = async (req: Request, res: Response) => { + try { + const { crawl_session_id } = CrawlSessionParamsSchema.parse(req.params); + await crawlService.deleteSession(crawl_session_id); + res.status(StatusCodes.OK).json({ message: 'Crawl session deleted successfully' }); + } catch (e) { + handleControllerError(res, e); + } +}; + +export const startSession = async (req: Request, res: Response) => { + try { + const { crawl_session_id } = CrawlSessionParamsSchema.parse(req.params); + await crawlService.startSession(crawl_session_id); + res.status(StatusCodes.OK).json({ message: 'Crawl session started successfully' }); + } catch (e) { + handleControllerError(res, e); + } +}; + +export const abortSession = async (req: Request, res: Response) => { + try { + const { crawl_session_id } = CrawlSessionParamsSchema.parse(req.params); + await crawlService.abortSession(crawl_session_id); + res.status(StatusCodes.OK).json({ message: 'Crawl session aborted successfully' }); + } catch (e) { + handleControllerError(res, e); + } +}; + +export const pauseSession = async (req: Request, res: Response) => { + try { + const { crawl_session_id } = CrawlSessionParamsSchema.parse(req.params); + await crawlService.pauseSession(crawl_session_id); + res.status(StatusCodes.OK).json({ message: 'Crawl session paused successfully' }); + } catch (e) { + handleControllerError(res, e); + } +}; \ No newline at end of file diff --git a/src/api/routes/crawlSession.routes.ts b/src/api/routes/crawlSession.routes.ts new file mode 100644 index 0000000..c248d83 --- /dev/null +++ b/src/api/routes/crawlSession.routes.ts @@ -0,0 +1,33 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + +import { Router, type RequestHandler } from 'express'; +import * as crawlerController from '../controllers/crawlSession.controller'; +import { requireAuth } from '../middlewares/requireAuth'; +import { validateBody } from '../middlewares/validate'; +import { CreateCrawlSessionRequestSchema } from '@models/crawlSession'; + +const router = Router(); + +router.get('/versions/:app_version_id/crawl-sessions', requireAuth, crawlerController.getSessions); + +router.post( + '/versions/:app_version_id/crawl-sessions', + requireAuth, + validateBody(CreateCrawlSessionRequestSchema), + crawlerController.createSession +); + + +router.get('/crawl-sessions/:crawl_session_id', requireAuth, crawlerController.getSessionDetails); + +router.delete('/crawl-sessions/:crawl_session_id', requireAuth, crawlerController.deleteSession); + +router.put('/crawl-sessions/:crawl_session_id/abort', requireAuth, crawlerController.abortSession); + +router.put('/crawl-sessions/:crawl_session_id/start', requireAuth, crawlerController.startSession); + +router.put('/crawl-sessions/:crawl_session_id/pause', requireAuth, crawlerController.pauseSession); + +export default router; \ No newline at end of file diff --git a/src/models/crawlSession.ts b/src/models/crawlSession.ts new file mode 100644 index 0000000..2adeb30 --- /dev/null +++ b/src/models/crawlSession.ts @@ -0,0 +1,65 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + +// CrawlSession domain DTOs + +import { + CrawlTriggerType, + CrawlStatus, + type CrawlConfig as ContractCrawlConfig, + type CreateCrawlSessionRequest as ContractCreateCrawlSessionRequest, + type CrawlSessionData as ContractCrawlSessionData, + type ApplicationVersionCrawlSessionsResponse as ContractApplicationVersionCrawlSessionsResponse, + type CrawlSessionByIDResponse as ContractCrawlSessionByIDResponse, + type StopCrawlSessionResponse as ContractStopCrawlSessionResponse, +} from "@coveritlabs/contracts"; +import { z } from "@utils/zod"; +import type { infer as ZodInfer, ZodType } from "zod"; +import type { Plain } from "./common"; + +export type CrawlConfig = Plain; +export type CreateCrawlSessionRequest = Plain; +export type CrawlSessionData = Plain; +export type ApplicationVersionCrawlSessionsResponse = Plain; +export type CrawlSessionByIDResponse = Plain; +export type StopCrawlSessionResponse = Plain; +export type GetSessionsQuery = ZodInfer; +export { CrawlTriggerType, CrawlStatus }; + + + + +export const CrawlConfigSchema = z.object({ + maxStates: z.number().int().min(1).max(100000), + maxDepth: z.number().int().min(1).max(1000), + includeUrlPatterns: z.array(z.string().min(1).max(2048)).max(100), + excludeUrlPatterns: z.array(z.string().min(1).max(2048)).max(100), + enableSemanticDecisions: z.boolean(), + headless: z.boolean(), + timeoutSeconds: z.number().int().min(1).max(86400), +}) satisfies ZodType; + +export const CreateCrawlSessionRequestSchema = z.object({ + triggerType: z.enum(CrawlTriggerType), + crawlConfig: CrawlConfigSchema, +}) satisfies ZodType; + +export const AppVersionParamsSchema = z.object({ + app_version_id: z.uuid(), +}); + +export const CrawlSessionParamsSchema = z.object({ + crawl_session_id: z.uuid(), +}); + +export const GetSessionsQuerySchema = z.object({ + page: z.coerce.number().int().min(1).default(1), + pageSize: z.coerce.number().int().min(1).max(100).default(25), + status: z.enum(CrawlStatus).optional(), + triggerType: z.enum(CrawlTriggerType).optional(), +}); + + + + diff --git a/src/queues/crawl.queue.ts b/src/queues/crawl.queue.ts new file mode 100644 index 0000000..8af2a43 --- /dev/null +++ b/src/queues/crawl.queue.ts @@ -0,0 +1,24 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + +import { Queue } from 'bullmq'; +import redis from '@lib/redis'; + +export const crawlQueue = new Queue('crawl-tasks', { + connection: redis +}); + +export async function addCrawlJob(sessionId: string) { + await crawlQueue.add('start-crawl', { sessionId }); +} + +export async function removeCrawlJob(sessionId: string) { + const jobs = await crawlQueue.getJobs(['waiting', 'delayed']); + const job = jobs.find((j) => j.data.sessionId === sessionId); + if (job) { + await job.remove(); + return true; + } + return false; +} \ No newline at end of file diff --git a/src/services/crawlSession.service.ts b/src/services/crawlSession.service.ts new file mode 100644 index 0000000..159739f --- /dev/null +++ b/src/services/crawlSession.service.ts @@ -0,0 +1,205 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + +import prisma from "@lib/prisma"; +import { + type CrawlConfig, + CrawlConfigSchema, + CrawlStatus, + CrawlTriggerType, + type ApplicationVersionCrawlSessionsResponse, + type CrawlSessionData, + type GetSessionsQuery, +} from "@models/crawlSession"; +import { removeCrawlJob, addCrawlJob } from "@queues/crawl.queue"; + + +type DbCrawlSession = Awaited>; + +const toIso = (value: Date | null): string | undefined => (value ? value.toISOString() : undefined); + +const toEnumValue = >( + enumObject: T, + raw: string, + fallback: T[keyof T], +): T[keyof T] => { + const value = enumObject[raw as keyof T]; + return typeof value === "number" ? value : fallback; +}; + +const crawlStatusToDb = (status?: CrawlStatus): string | undefined => { + if (status === undefined || status === CrawlStatus.UNSPECIFIED) { + return undefined; + } + return CrawlStatus[status]; +}; + +const crawlTriggerTypeToDb = (triggerType?: CrawlTriggerType): string | undefined => { + if (triggerType === undefined || triggerType === CrawlTriggerType.UNSPECIFIED) { + return undefined; + } + return CrawlTriggerType[triggerType]; +}; + +const mapSession = (session: DbCrawlSession): CrawlSessionData => ({ + id: session.id, + appVersionId: session.appVersionId, + status: toEnumValue(CrawlStatus, session.status, CrawlStatus.UNSPECIFIED) as CrawlStatus, + triggerType: toEnumValue(CrawlTriggerType, session.triggerType, CrawlTriggerType.UNSPECIFIED) as CrawlTriggerType, + crawlConfig: (() => { + const parsed = CrawlConfigSchema.safeParse(session.config); + return parsed.success ? parsed.data : undefined; + })(), + stateCount: session.stateCount, + transitionCount: session.transitionCount, + createdAt: session.createdAt.toISOString(), + startedAt: toIso(session.startedAt), + finishedAt: toIso(session.finishedAt), + errorMessage: session.error ?? undefined, +}); + +export async function getSessions( + versionId: string, + query: GetSessionsQuery +): Promise { + const { page, pageSize, status, triggerType } = query; + const dbStatus = crawlStatusToDb(status); + const dbTriggerType = crawlTriggerTypeToDb(triggerType); + const [sessions, totalCount] = await Promise.all([ + prisma.crawlSession.findMany({ + where: { + appVersionId: versionId, + status: dbStatus, + triggerType: dbTriggerType, + }, + orderBy: { createdAt: 'desc' }, + skip: (page - 1) * pageSize, + take: pageSize, + }), + prisma.crawlSession.count({ + where: { + appVersionId: versionId, + status: dbStatus, + triggerType: dbTriggerType, + } + }) + ]); + + return { + sessions: sessions.map(mapSession), + totalCount: totalCount, + currentPage: page, + pageSize: pageSize, + }; +} + +export async function createSession( + versionId: string, + triggerType: CrawlTriggerType, + crawlConfig: CrawlConfig, +): Promise { + const parsedConfig = CrawlConfigSchema.parse(crawlConfig); + const persistedConfig = { + maxStates: parsedConfig.maxStates, + maxDepth: parsedConfig.maxDepth, + includeUrlPatterns: parsedConfig.includeUrlPatterns, + excludeUrlPatterns: parsedConfig.excludeUrlPatterns, + enableSemanticDecisions: parsedConfig.enableSemanticDecisions, + headless: parsedConfig.headless, + timeoutSeconds: parsedConfig.timeoutSeconds, + }; + + const newSession = await prisma.crawlSession.create({ + data: { + appVersionId: versionId, + triggerType: CrawlTriggerType[triggerType], + config: persistedConfig, + } + }); + return mapSession(newSession); +} + +export async function getSessionDetails(sessionId: string): Promise { + const session = await prisma.crawlSession.findUniqueOrThrow({ + where: { id: sessionId } + }); + return mapSession(session); +} + +export async function deleteSession(sessionId: string): Promise { + await removeCrawlJob(sessionId); + await prisma.crawlSession.delete({ + where: { id: sessionId } + }); +} + +export async function startSession(sessionId: string): Promise { + const session = await prisma.crawlSession.findUniqueOrThrow({ + where: { id: sessionId } + }); + + if (session.status === CrawlStatus[CrawlStatus.NEW]) { + await prisma.crawlSession.update({ + where: { id: sessionId }, + data: { status: CrawlStatus[CrawlStatus.QUEUED] } + }); + + try { + await addCrawlJob(sessionId); + } catch (error) { + await prisma.crawlSession.update({ + where: { id: sessionId }, + data: { status: CrawlStatus[CrawlStatus.NEW] } + }); + throw error; + } + + return; + } + + if (session.status === CrawlStatus[CrawlStatus.PAUSED]) { + await prisma.crawlSession.update({ + where: { id: sessionId }, + data: { status: CrawlStatus[CrawlStatus.RUNNING] } + }); + return; + } + + throw new Error(`Cannot start session with status ${session.status}`); +}; + +export async function abortSession(sessionId: string): Promise { + const session = await prisma.crawlSession.findUniqueOrThrow({ + where: { id: sessionId } + }); + + if (session.status !== CrawlStatus[CrawlStatus.RUNNING] && session.status !== CrawlStatus[CrawlStatus.PAUSED] + && session.status !== CrawlStatus[CrawlStatus.QUEUED]) { + throw new Error(`Cannot abort session with status ${session.status}`); + } + + if (session.status === CrawlStatus[CrawlStatus.QUEUED]) { + await removeCrawlJob(sessionId); + } + + await prisma.crawlSession.update({ + where: { id: sessionId }, + data: { status: CrawlStatus[CrawlStatus.ABORTED] } + }); +}; + +export async function pauseSession(sessionId: string): Promise { + const session = await prisma.crawlSession.findUniqueOrThrow({ + where: { id: sessionId } + }); + + if (session.status !== CrawlStatus[CrawlStatus.RUNNING]) { + throw new Error(`Cannot pause session with status ${session.status}`); + } + + await prisma.crawlSession.update({ + where: { id: sessionId }, + data: { status: CrawlStatus[CrawlStatus.PAUSED] } + }); +}; diff --git a/src/workers/crawler.worker.ts b/src/workers/crawler.worker.ts new file mode 100644 index 0000000..9356877 --- /dev/null +++ b/src/workers/crawler.worker.ts @@ -0,0 +1,139 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + +import { Worker } from 'bullmq'; +import { spawn } from 'child_process'; +import redis from '@lib/redis'; +import prisma from '@lib/prisma'; +import { CrawlStatus } from '@models/crawlSession'; + +export const crawlWorker = new Worker('crawl-tasks', async (job) => { + const { sessionId } = job.data; + // only do it if its status is QUEUED, otherwise it means that the session was aborted while it was in the queue, and we should not start it. + const session = await prisma.crawlSession.findFirstOrThrow({ + where: { id: sessionId } + }); + if (session.status !== CrawlStatus[CrawlStatus.QUEUED]) { + throw new Error(`Cannot start session with status ${session.status}`); + } + + await prisma.crawlSession.update({ + where: { id: sessionId }, + data: { + status: CrawlStatus[CrawlStatus.RUNNING], + startedAt: new Date(), + }, + }); + + const pythonBinary = process.env.CRAWLER_PYTHON_BIN ?? 'python3'; + + const pythonProcess = spawn(pythonBinary, ['main.py', '--session', sessionId], { + stdio: ['pipe', 'pipe', 'pipe'], + cwd: 'src/workers', + detached: false, + }); + + const pid = pythonProcess.pid; + if (pid === undefined) { + throw new Error(`Failed to start crawler process for session ${sessionId}`); + } + await redis.set(`session:${sessionId}:pid`, String(pid), 'EX', 86400); + + const cleanup = async () => { + await redis.del(`session:${sessionId}:pid`); + }; + + pythonProcess.once('close', () => { + void cleanup(); + }); + + pythonProcess.stdout.on('data', (data) => { + console.log(`[Session ${sessionId}] Python: ${data}`); + }); + + pythonProcess.stderr.on('data', (data) => { + console.error(`[Session ${sessionId}] Error: ${data}`); + }); + + let deleted = false; + + const exitCode = await new Promise((resolve) => { + let lastStatus = CrawlStatus[CrawlStatus.RUNNING]; + let checking = false; + + const checkInterval = setInterval(async () => { + if (checking) { + return; + } + + checking = true; + let current; + try { + current = await prisma.crawlSession.findFirstOrThrow({ + where: { id: sessionId }, + select: { status: true } + }); + } catch (e) { + // this means that the session was deleted while the crawl was running. In this case, we should stop the crawl immediately. + pythonProcess.stdin.write('ABORT\n'); + clearInterval(checkInterval); + setTimeout(() => pythonProcess.kill('SIGKILL'), 2000); + deleted = true; + return; + } + + checking = false; + + if (!current) return; + if (current.status === CrawlStatus[CrawlStatus.ABORTED]) { + pythonProcess.stdin.write('ABORT\n'); + clearInterval(checkInterval); + setTimeout(() => pythonProcess.kill('SIGKILL'), 2000); + return; + } + + if (current.status !== lastStatus) { + if (current.status === CrawlStatus[CrawlStatus.PAUSED]) { + pythonProcess.stdin.write('PAUSE\n'); + } else if (current.status === CrawlStatus[CrawlStatus.RUNNING]) { + pythonProcess.stdin.write('RESUME\n'); + } + lastStatus = current.status; + } + }, 3000); + + pythonProcess.on('close', (code) => { + clearInterval(checkInterval); + resolve(code); + }); + + pythonProcess.on('error', (err) => { + console.error("Failed to start child process:", err); + clearInterval(checkInterval); + resolve(1); + }); + }); + + if (!deleted) { + const finalSession = await prisma.crawlSession.findUnique({ where: { id: sessionId } }); + const isAborted = finalSession?.status === CrawlStatus[CrawlStatus.ABORTED]; + const isStopped = finalSession?.status === CrawlStatus[CrawlStatus.PAUSED]; + + if (exitCode === 0) { + if (!isAborted && !isStopped) { + await prisma.crawlSession.update({ + where: { id: sessionId }, + data: { status: CrawlStatus[CrawlStatus.COMPLETED], finishedAt: new Date() } + }); + } + } else { + if (!isAborted) { + await prisma.crawlSession.update({ + where: { id: sessionId }, + data: { status: CrawlStatus[CrawlStatus.FAILED], finishedAt: new Date() } + }); + } + } + } +}, { connection: redis }); \ No newline at end of file diff --git a/src/workers/fake_crawler.py b/src/workers/fake_crawler.py new file mode 100644 index 0000000..c930a27 --- /dev/null +++ b/src/workers/fake_crawler.py @@ -0,0 +1,44 @@ +# Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +# Proprietary and confidential. Unauthorized use is strictly prohibited. +# See LICENSE file in the project root for full license information. + +# this is just a mimic and fake crawler worker that simulates a crawl session for testing purposes. It does not perform any real crawling. +import argparse +import sys +import threading +import time + +# Control state +state = {"paused": False, "running": True} + +def command_listener(): + for line in sys.stdin: + cmd = line.strip() + if cmd == "ABORT": + state["running"] = False + break + elif cmd == "PAUSE": + state["paused"] = True + elif cmd == "RESUME": + state["paused"] = False + +# Start background listener +threading.Thread(target=command_listener, daemon=True).start() + +def run_crawler(): + while state["running"]: + if state["paused"]: + time.sleep(1) # Wait and check again + continue + print(f"Crawling... {i+1}/30") + time.sleep(1) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Fake Crawler Worker') + parser.add_argument('--session', type=str, required=True, help='Crawl session ID') + args = parser.parse_args() + session_id = args.session + print(f"Starting fake crawl session: {session_id}") + run_crawler() + print(f"Finished fake crawl session: {session_id}") \ No newline at end of file From 8c4741055d5a0c825843233eb70044908f0edfb4 Mon Sep 17 00:00:00 2001 From: Youssef Date: Sat, 11 Apr 2026 20:01:31 +0200 Subject: [PATCH 2/6] feat: add crawler worker --- package.json | 2 + .../4_add_crawl_sessions/migration.sql | 47 +++ prisma/schema.prisma | 23 +- .../controllers/crawlSession.controller.ts | 28 +- src/api/routes/crawlSession.routes.ts | 34 +- src/api/routes/targetApplication.routes.ts | 3 + src/lib/cache.ts | 3 + src/models/crawlSession.ts | 45 ++- src/queues/crawl.queue.ts | 2 +- src/services/crawlSession.service.ts | 184 +++++++++-- src/types/crawler.ts | 31 ++ src/utils/date.ts | 7 + src/workers/crawler.worker.ts | 293 +++++++++++------- 13 files changed, 519 insertions(+), 183 deletions(-) create mode 100644 prisma/migrations/4_add_crawl_sessions/migration.sql create mode 100644 src/types/crawler.ts create mode 100644 src/utils/date.ts diff --git a/package.json b/package.json index 31f9e78..71d6e4a 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,8 @@ "dev": "nodemon", "build": "prisma generate && tsc && tsc-alias", "start": "node dist/index.js", + "worker:email": "ts-node -r tsconfig-paths/register src/workers/email.worker.ts", + "worker:crawler": "ts-node -r tsconfig-paths/register src/workers/crawler.worker.ts", "postinstall": "prisma generate", "db:migrate": "prisma migrate dev", "db:deploy": "prisma migrate deploy", diff --git a/prisma/migrations/4_add_crawl_sessions/migration.sql b/prisma/migrations/4_add_crawl_sessions/migration.sql new file mode 100644 index 0000000..ba969cb --- /dev/null +++ b/prisma/migrations/4_add_crawl_sessions/migration.sql @@ -0,0 +1,47 @@ +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'CrawlStatus') THEN + CREATE TYPE "CrawlStatus" AS ENUM ( + 'UNSPECIFIED', + 'QUEUED', + 'RUNNING', + 'COMPLETED', + 'FAILED', + 'ABORTED', + 'PAUSED', + 'NEW' + ); + END IF; +END $$; + +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'CrawlTriggerType') THEN + CREATE TYPE "CrawlTriggerType" AS ENUM ( + 'UNSPECIFIED', + 'MANUAL', + 'SCHEDULED', + 'CI_TRIGGER', + 'WEBHOOK' + ); + END IF; +END $$; + +CREATE TABLE IF NOT EXISTS "crawl_sessions" ( + "crawl_session_id" UUID NOT NULL, + "app_version_id" UUID NOT NULL, + "status" "CrawlStatus" NOT NULL DEFAULT 'NEW', + "trigger_type" "CrawlTriggerType" NOT NULL, + "config" JSONB NOT NULL, + "state_count" INTEGER NOT NULL DEFAULT 0, + "transition_count" INTEGER NOT NULL DEFAULT 0, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "started_at" TIMESTAMP(3), + "finished_at" TIMESTAMP(3), + "error" TEXT, + CONSTRAINT "crawl_sessions_pkey" PRIMARY KEY ("crawl_session_id") +); + +ALTER TABLE "crawl_sessions" + ADD CONSTRAINT "crawl_sessions_app_version_id_fkey" + FOREIGN KEY ("app_version_id") + REFERENCES "target_application_versions"("id") + ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e8280f0..53c9aa4 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -62,6 +62,25 @@ enum ProjectRole { VIEWER } +enum CrawlStatus { + UNSPECIFIED + QUEUED + RUNNING + COMPLETED + FAILED + ABORTED + PAUSED + NEW +} + +enum CrawlTriggerType { + UNSPECIFIED + MANUAL + SCHEDULED + CI_TRIGGER + WEBHOOK +} + model ProjectMember { id String @id @default(uuid()) @db.Uuid projectId String @map("project_id") @db.Uuid @@ -131,8 +150,8 @@ model RegressionCodebase { model CrawlSession { id String @id @default(uuid()) @map("crawl_session_id") @db.Uuid appVersionId String @map("app_version_id") @db.Uuid - status String @default("NEW") - triggerType String @map("trigger_type") + status CrawlStatus @default(NEW) + triggerType CrawlTriggerType @map("trigger_type") config Json stateCount Int @default(0) @map("state_count") transitionCount Int @default(0) @map("transition_count") diff --git a/src/api/controllers/crawlSession.controller.ts b/src/api/controllers/crawlSession.controller.ts index 7a15c87..bff693b 100644 --- a/src/api/controllers/crawlSession.controller.ts +++ b/src/api/controllers/crawlSession.controller.ts @@ -39,9 +39,9 @@ function handleControllerError(res: Response, error: unknown): void { export const getSessions = async (req: Request, res: Response) => { try { - const { app_version_id } = AppVersionParamsSchema.parse(req.params); + const { versionId } = AppVersionParamsSchema.parse(req.params); const query = GetSessionsQuerySchema.parse(req.query); - const result = await crawlService.getSessions(app_version_id, query); + const result = await crawlService.getSessions(versionId, query); res.json(result); } catch (e) { handleControllerError(res, e); @@ -50,9 +50,9 @@ export const getSessions = async (req: Request, res: Response) => { export const createSession = async (req: Request, res: Response) => { try { - const { app_version_id } = AppVersionParamsSchema.parse(req.params); + const { versionId } = AppVersionParamsSchema.parse(req.params); const body = CreateCrawlSessionRequestSchema.parse(req.body); - const result = await crawlService.createSession(app_version_id, body.triggerType, body.crawlConfig); + const result = await crawlService.createSession(versionId, body.triggerType, body.crawlConfig); res.status(StatusCodes.CREATED).json(result); } catch (e) { handleControllerError(res, e); @@ -61,8 +61,8 @@ export const createSession = async (req: Request, res: Response) => { export const getSessionDetails = async (req: Request, res: Response) => { try { - const { crawl_session_id } = CrawlSessionParamsSchema.parse(req.params); - const result = await crawlService.getSessionDetails(crawl_session_id); + const { crawlSessionId } = CrawlSessionParamsSchema.parse(req.params); + const result = await crawlService.getSessionDetails(crawlSessionId); res.json(result); } catch (e) { handleControllerError(res, e); @@ -71,8 +71,8 @@ export const getSessionDetails = async (req: Request, res: Response) => { export const deleteSession = async (req: Request, res: Response) => { try { - const { crawl_session_id } = CrawlSessionParamsSchema.parse(req.params); - await crawlService.deleteSession(crawl_session_id); + const { crawlSessionId } = CrawlSessionParamsSchema.parse(req.params); + await crawlService.deleteSession(crawlSessionId); res.status(StatusCodes.OK).json({ message: 'Crawl session deleted successfully' }); } catch (e) { handleControllerError(res, e); @@ -81,8 +81,8 @@ export const deleteSession = async (req: Request, res: Response) => { export const startSession = async (req: Request, res: Response) => { try { - const { crawl_session_id } = CrawlSessionParamsSchema.parse(req.params); - await crawlService.startSession(crawl_session_id); + const { crawlSessionId } = CrawlSessionParamsSchema.parse(req.params); + await crawlService.startSession(crawlSessionId); res.status(StatusCodes.OK).json({ message: 'Crawl session started successfully' }); } catch (e) { handleControllerError(res, e); @@ -91,8 +91,8 @@ export const startSession = async (req: Request, res: Response) => { export const abortSession = async (req: Request, res: Response) => { try { - const { crawl_session_id } = CrawlSessionParamsSchema.parse(req.params); - await crawlService.abortSession(crawl_session_id); + const { crawlSessionId } = CrawlSessionParamsSchema.parse(req.params); + await crawlService.abortSession(crawlSessionId); res.status(StatusCodes.OK).json({ message: 'Crawl session aborted successfully' }); } catch (e) { handleControllerError(res, e); @@ -101,8 +101,8 @@ export const abortSession = async (req: Request, res: Response) => { export const pauseSession = async (req: Request, res: Response) => { try { - const { crawl_session_id } = CrawlSessionParamsSchema.parse(req.params); - await crawlService.pauseSession(crawl_session_id); + const { crawlSessionId } = CrawlSessionParamsSchema.parse(req.params); + await crawlService.pauseSession(crawlSessionId); res.status(StatusCodes.OK).json({ message: 'Crawl session paused successfully' }); } catch (e) { handleControllerError(res, e); diff --git a/src/api/routes/crawlSession.routes.ts b/src/api/routes/crawlSession.routes.ts index c248d83..537241d 100644 --- a/src/api/routes/crawlSession.routes.ts +++ b/src/api/routes/crawlSession.routes.ts @@ -2,32 +2,30 @@ // Proprietary and confidential. Unauthorized use is strictly prohibited. // See LICENSE file in the project root for full license information. -import { Router, type RequestHandler } from 'express'; -import * as crawlerController from '../controllers/crawlSession.controller'; -import { requireAuth } from '../middlewares/requireAuth'; -import { validateBody } from '../middlewares/validate'; -import { CreateCrawlSessionRequestSchema } from '@models/crawlSession'; +import { Router } from "express"; -const router = Router(); +import * as crawlSessionController from "@api/controllers/crawlSession.controller"; +import { requireAuth } from "@api/middlewares/requireAuth"; +import { requireProjectAdmin, requireProjectMember, requireProjectMembership } from "@api/middlewares/requireProjectAccess"; +import { validateBody } from "@api/middlewares/validate"; +import { CreateCrawlSessionRequestSchema } from "@models/crawlSession"; -router.get('/versions/:app_version_id/crawl-sessions', requireAuth, crawlerController.getSessions); +const router = Router({ mergeParams: true }); -router.post( - '/versions/:app_version_id/crawl-sessions', - requireAuth, - validateBody(CreateCrawlSessionRequestSchema), - crawlerController.createSession -); +router.use(requireAuth); +router.get("/", requireProjectMembership, crawlSessionController.getSessions); -router.get('/crawl-sessions/:crawl_session_id', requireAuth, crawlerController.getSessionDetails); +router.post("/", requireProjectMember, validateBody(CreateCrawlSessionRequestSchema), crawlSessionController.createSession); -router.delete('/crawl-sessions/:crawl_session_id', requireAuth, crawlerController.deleteSession); +router.get("/:crawlSessionId", requireProjectMembership, crawlSessionController.getSessionDetails); -router.put('/crawl-sessions/:crawl_session_id/abort', requireAuth, crawlerController.abortSession); +router.delete("/:crawlSessionId", requireProjectAdmin, crawlSessionController.deleteSession); -router.put('/crawl-sessions/:crawl_session_id/start', requireAuth, crawlerController.startSession); +router.put("/:crawlSessionId/abort", requireProjectMember, crawlSessionController.abortSession); -router.put('/crawl-sessions/:crawl_session_id/pause', requireAuth, crawlerController.pauseSession); +router.put("/:crawlSessionId/start", requireProjectMember, crawlSessionController.startSession); + +router.put("/:crawlSessionId/pause", requireProjectMember, crawlSessionController.pauseSession); export default router; \ No newline at end of file diff --git a/src/api/routes/targetApplication.routes.ts b/src/api/routes/targetApplication.routes.ts index a72c924..2b9e048 100644 --- a/src/api/routes/targetApplication.routes.ts +++ b/src/api/routes/targetApplication.routes.ts @@ -14,6 +14,7 @@ import { CreateTargetApplicationVersionRequestSchema, } from "@models/targetApplication"; import { CreateRegressionCodebaseRequestSchema, UpdateRegressionCodebaseRequestSchema } from "@models/regressionCodebase"; +import crawlSessionRoutes from "@api/routes/crawlSession.routes"; const router = Router({ mergeParams: true }); @@ -28,6 +29,8 @@ router.get("/:appId", requireProjectMembership, targetController.getTargetApplic router.post("/:appId/versions", requireProjectMember, validateBody(CreateTargetApplicationVersionRequestSchema), targetController.createVersion); router.delete("/:appId/versions/:versionId", requireProjectAdmin, targetController.deleteVersion); +router.use("/:appId/versions/:versionId/crawl-sessions", crawlSessionRoutes); + // Regression codebases router.post( "/:appId/regression-codebases", diff --git a/src/lib/cache.ts b/src/lib/cache.ts index 2dea045..01ee94c 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -64,6 +64,9 @@ export const cacheKeys = { byId: (codebaseId: string): string => `regression_codebase:${codebaseId}`, byApp: (appId: string): string => `app:regression_codebases:${appId}`, }, + crawlSession: { + pid: (sessionId: string): string => `session:${sessionId}:pid`, + }, } as const; export async function cacheGetJSON(key: string, context?: string): Promise { diff --git a/src/models/crawlSession.ts b/src/models/crawlSession.ts index 2adeb30..066680a 100644 --- a/src/models/crawlSession.ts +++ b/src/models/crawlSession.ts @@ -17,8 +17,12 @@ import { import { z } from "@utils/zod"; import type { infer as ZodInfer, ZodType } from "zod"; import type { Plain } from "./common"; +import { type InputDefaultsConfig, type CrawlerRunSettings } from "types/crawler"; -export type CrawlConfig = Plain; +export type CrawlConfig = Plain & { + crawlerSettings?: CrawlerRunSettings; + inputDefaults?: InputDefaultsConfig; +}; export type CreateCrawlSessionRequest = Plain; export type CrawlSessionData = Plain; export type ApplicationVersionCrawlSessionsResponse = Plain; @@ -38,26 +42,53 @@ export const CrawlConfigSchema = z.object({ enableSemanticDecisions: z.boolean(), headless: z.boolean(), timeoutSeconds: z.number().int().min(1).max(86400), -}) satisfies ZodType; + crawlerSettings: z + .object({ + headless: z.boolean().optional(), + timeout_ms: z.number().int().min(1).max(86400_000).optional(), + max_states: z.number().int().min(1).max(100000).optional(), + max_transitions: z.number().int().min(1).max(1_000_000).optional(), + max_elements_per_state: z.number().int().min(1).max(10000).optional(), + max_select_options_per_element: z.number().int().min(1).max(1000).optional(), + max_action_repeats_per_url: z.number().int().min(0).max(1000).optional(), + action_retry_count: z.number().int().min(0).max(100).optional(), + replay_retry_count: z.number().int().min(0).max(100).optional(), + popup_timeout_ms: z.number().int().min(1).max(86400_000).optional(), + dom_quiet_ms: z.number().int().min(0).max(600_000).optional(), + dom_settle_timeout_ms: z.number().int().min(1).max(86400_000).optional(), + use_dom_quiescence: z.boolean().optional(), + page_load_state: z.string().min(1).max(100).optional(), + click_non_http_links: z.boolean().optional(), + defer_destructive_actions: z.boolean().optional(), + destructive_keywords: z.string().min(0).max(5000).optional(), + }) + .optional(), + inputDefaults: z + .object({ + field_patterns: z.record(z.string(), z.string()), + type_fallbacks: z.record(z.string(), z.string()), + }) + .optional(), +}).loose() satisfies ZodType; export const CreateCrawlSessionRequestSchema = z.object({ - triggerType: z.enum(CrawlTriggerType), + triggerType: z.nativeEnum(CrawlTriggerType), crawlConfig: CrawlConfigSchema, }) satisfies ZodType; export const AppVersionParamsSchema = z.object({ - app_version_id: z.uuid(), + versionId: z.uuid(), }); export const CrawlSessionParamsSchema = z.object({ - crawl_session_id: z.uuid(), + crawlSessionId: z.uuid(), }); export const GetSessionsQuerySchema = z.object({ page: z.coerce.number().int().min(1).default(1), pageSize: z.coerce.number().int().min(1).max(100).default(25), - status: z.enum(CrawlStatus).optional(), - triggerType: z.enum(CrawlTriggerType).optional(), + status: z.nativeEnum(CrawlStatus).optional(), + triggerType: z.nativeEnum(CrawlTriggerType).optional(), }); diff --git a/src/queues/crawl.queue.ts b/src/queues/crawl.queue.ts index 8af2a43..d459dd7 100644 --- a/src/queues/crawl.queue.ts +++ b/src/queues/crawl.queue.ts @@ -10,7 +10,7 @@ export const crawlQueue = new Queue('crawl-tasks', { }); export async function addCrawlJob(sessionId: string) { - await crawlQueue.add('start-crawl', { sessionId }); + await crawlQueue.add('crawl', { sessionId }); } export async function removeCrawlJob(sessionId: string) { diff --git a/src/services/crawlSession.service.ts b/src/services/crawlSession.service.ts index 159739f..7da6b3d 100644 --- a/src/services/crawlSession.service.ts +++ b/src/services/crawlSession.service.ts @@ -13,40 +13,40 @@ import { type GetSessionsQuery, } from "@models/crawlSession"; import { removeCrawlJob, addCrawlJob } from "@queues/crawl.queue"; +import { CrawlerJobPayload } from "types/crawler"; +import { toIso } from "@utils/date"; + + type DbCrawlSession = Awaited>; -const toIso = (value: Date | null): string | undefined => (value ? value.toISOString() : undefined); +type DbCrawlStatus = DbCrawlSession["status"]; +type DbCrawlTriggerType = DbCrawlSession["triggerType"]; -const toEnumValue = >( - enumObject: T, - raw: string, - fallback: T[keyof T], -): T[keyof T] => { - const value = enumObject[raw as keyof T]; - return typeof value === "number" ? value : fallback; +const toDbStatusFilter = (status?: CrawlStatus): DbCrawlStatus | undefined => { + if (status === undefined || status === CrawlStatus.UNSPECIFIED) return undefined; + return CrawlStatus[status] as unknown as DbCrawlStatus; }; -const crawlStatusToDb = (status?: CrawlStatus): string | undefined => { - if (status === undefined || status === CrawlStatus.UNSPECIFIED) { - return undefined; - } - return CrawlStatus[status]; +const toDbTriggerTypeFilter = (triggerType?: CrawlTriggerType): DbCrawlTriggerType | undefined => { + if (triggerType === undefined || triggerType === CrawlTriggerType.UNSPECIFIED) return undefined; + return CrawlTriggerType[triggerType] as unknown as DbCrawlTriggerType; }; -const crawlTriggerTypeToDb = (triggerType?: CrawlTriggerType): string | undefined => { - if (triggerType === undefined || triggerType === CrawlTriggerType.UNSPECIFIED) { - return undefined; - } - return CrawlTriggerType[triggerType]; +const fromDbStatus = (status: DbCrawlStatus): CrawlStatus => { + return (CrawlStatus[status as unknown as keyof typeof CrawlStatus] ?? CrawlStatus.UNSPECIFIED) as CrawlStatus; +}; + +const fromDbTriggerType = (triggerType: DbCrawlTriggerType): CrawlTriggerType => { + return (CrawlTriggerType[triggerType as unknown as keyof typeof CrawlTriggerType] ?? CrawlTriggerType.UNSPECIFIED) as CrawlTriggerType; }; const mapSession = (session: DbCrawlSession): CrawlSessionData => ({ id: session.id, appVersionId: session.appVersionId, - status: toEnumValue(CrawlStatus, session.status, CrawlStatus.UNSPECIFIED) as CrawlStatus, - triggerType: toEnumValue(CrawlTriggerType, session.triggerType, CrawlTriggerType.UNSPECIFIED) as CrawlTriggerType, + status: fromDbStatus(session.status), + triggerType: fromDbTriggerType(session.triggerType), crawlConfig: (() => { const parsed = CrawlConfigSchema.safeParse(session.config); return parsed.success ? parsed.data : undefined; @@ -64,8 +64,8 @@ export async function getSessions( query: GetSessionsQuery ): Promise { const { page, pageSize, status, triggerType } = query; - const dbStatus = crawlStatusToDb(status); - const dbTriggerType = crawlTriggerTypeToDb(triggerType); + const dbStatus = toDbStatusFilter(status); + const dbTriggerType = toDbTriggerTypeFilter(triggerType); const [sessions, totalCount] = await Promise.all([ prisma.crawlSession.findMany({ where: { @@ -108,12 +108,14 @@ export async function createSession( enableSemanticDecisions: parsedConfig.enableSemanticDecisions, headless: parsedConfig.headless, timeoutSeconds: parsedConfig.timeoutSeconds, + crawlerSettings: parsedConfig.crawlerSettings, + inputDefaults: parsedConfig.inputDefaults, }; const newSession = await prisma.crawlSession.create({ data: { appVersionId: versionId, - triggerType: CrawlTriggerType[triggerType], + triggerType: CrawlTriggerType[triggerType] as unknown as DbCrawlTriggerType, config: persistedConfig, } }); @@ -203,3 +205,139 @@ export async function pauseSession(sessionId: string): Promise { data: { status: CrawlStatus[CrawlStatus.PAUSED] } }); }; + +export async function markQueuedSessionRunning(sessionId: string): Promise { + const result = await prisma.crawlSession.updateMany({ + where: { + id: sessionId, + status: CrawlStatus[CrawlStatus.QUEUED], + }, + data: { + status: CrawlStatus[CrawlStatus.RUNNING], + startedAt: new Date(), + }, + }); + + if (result.count === 1) return; + + const session = await prisma.crawlSession.findUniqueOrThrow({ + where: { id: sessionId }, + }); + throw new Error(`Cannot start session with status ${session.status}`); +} + +export async function markSessionCompleted(sessionId: string): Promise { + await prisma.crawlSession.update({ + where: { id: sessionId }, + data: { + status: CrawlStatus[CrawlStatus.COMPLETED], + finishedAt: new Date(), + error: null, + }, + }); +} + +export async function markSessionFailed(sessionId: string, errorMessage: string): Promise { + await prisma.crawlSession.update({ + where: { id: sessionId }, + data: { + status: CrawlStatus[CrawlStatus.FAILED], + finishedAt: new Date(), + error: errorMessage, + }, + }); +} + +export async function isSessionAborted(sessionId: string): Promise { + const session = await prisma.crawlSession.findUnique({ + where: { id: sessionId }, + select: { status: true }, + }); + return session?.status === CrawlStatus[CrawlStatus.ABORTED]; +} + +export async function markSessionCompletedIfRunning(sessionId: string): Promise { + const result = await prisma.crawlSession.updateMany({ + where: { + id: sessionId, + status: CrawlStatus[CrawlStatus.RUNNING], + }, + data: { + status: CrawlStatus[CrawlStatus.COMPLETED], + finishedAt: new Date(), + error: null, + }, + }); + return result.count === 1; +} + +export async function markSessionFailedIfRunning(sessionId: string, errorMessage: string): Promise { + const result = await prisma.crawlSession.updateMany({ + where: { + id: sessionId, + status: CrawlStatus[CrawlStatus.RUNNING], + }, + data: { + status: CrawlStatus[CrawlStatus.FAILED], + finishedAt: new Date(), + error: errorMessage, + }, + }); + return result.count === 1; +} + +export async function markSessionFinishedAtIfAborted(sessionId: string): Promise { + await prisma.crawlSession.updateMany({ + where: { + id: sessionId, + status: CrawlStatus[CrawlStatus.ABORTED], + finishedAt: null, + }, + data: { + finishedAt: new Date(), + }, + }); +} + +export async function getCrawlerJobPayload(sessionId: string): Promise { + const session = await prisma.crawlSession.findUniqueOrThrow({ + where: { id: sessionId }, + include: { + appVersion: { + include: { + targetApplication: true, + }, + }, + }, + }); + + const parsed = CrawlConfigSchema.safeParse(session.config); + const crawlConfig = parsed.success ? parsed.data : undefined; + + const settings = { + headless: crawlConfig?.crawlerSettings?.headless ?? crawlConfig?.headless, + timeout_ms: crawlConfig?.crawlerSettings?.timeout_ms ?? (crawlConfig?.timeoutSeconds ? crawlConfig.timeoutSeconds * 1000 : undefined), + max_states: crawlConfig?.crawlerSettings?.max_states ?? crawlConfig?.maxStates, + max_transitions: crawlConfig?.crawlerSettings?.max_transitions, + max_elements_per_state: crawlConfig?.crawlerSettings?.max_elements_per_state, + max_select_options_per_element: crawlConfig?.crawlerSettings?.max_select_options_per_element, + max_action_repeats_per_url: crawlConfig?.crawlerSettings?.max_action_repeats_per_url, + action_retry_count: crawlConfig?.crawlerSettings?.action_retry_count, + replay_retry_count: crawlConfig?.crawlerSettings?.replay_retry_count, + popup_timeout_ms: crawlConfig?.crawlerSettings?.popup_timeout_ms, + dom_quiet_ms: crawlConfig?.crawlerSettings?.dom_quiet_ms, + dom_settle_timeout_ms: crawlConfig?.crawlerSettings?.dom_settle_timeout_ms, + use_dom_quiescence: crawlConfig?.crawlerSettings?.use_dom_quiescence, + page_load_state: crawlConfig?.crawlerSettings?.page_load_state, + click_non_http_links: crawlConfig?.crawlerSettings?.click_non_http_links, + defer_destructive_actions: crawlConfig?.crawlerSettings?.defer_destructive_actions, + destructive_keywords: crawlConfig?.crawlerSettings?.destructive_keywords, + }; + + return { + base_url: session.appVersion.targetApplication.baseUrl, + session_id: sessionId, + settings, + input_defaults: crawlConfig?.inputDefaults, + }; +} diff --git a/src/types/crawler.ts b/src/types/crawler.ts new file mode 100644 index 0000000..fcce1e8 --- /dev/null +++ b/src/types/crawler.ts @@ -0,0 +1,31 @@ +export type InputDefaultsConfig = { + field_patterns: Record; + type_fallbacks: Record; +}; + +export type CrawlerRunSettings = { + headless?: boolean; + timeout_ms?: number; + max_states?: number; + max_transitions?: number; + max_elements_per_state?: number; + max_select_options_per_element?: number; + max_action_repeats_per_url?: number; + action_retry_count?: number; + replay_retry_count?: number; + popup_timeout_ms?: number; + dom_quiet_ms?: number; + dom_settle_timeout_ms?: number; + use_dom_quiescence?: boolean; + page_load_state?: string; + click_non_http_links?: boolean; + defer_destructive_actions?: boolean; + destructive_keywords?: string; +}; + +export type CrawlerJobPayload = { + base_url: string; + session_id: string; + settings: CrawlerRunSettings; + input_defaults?: InputDefaultsConfig; +}; \ No newline at end of file diff --git a/src/utils/date.ts b/src/utils/date.ts new file mode 100644 index 0000000..72787ee --- /dev/null +++ b/src/utils/date.ts @@ -0,0 +1,7 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + +export function toIso(value: Date | null): string | undefined { + return value ? value.toISOString() : undefined; +} diff --git a/src/workers/crawler.worker.ts b/src/workers/crawler.worker.ts index 9356877..2a79882 100644 --- a/src/workers/crawler.worker.ts +++ b/src/workers/crawler.worker.ts @@ -1,139 +1,196 @@ -// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. -// Proprietary and confidential. Unauthorized use is strictly prohibited. -// See LICENSE file in the project root for full license information. - -import { Worker } from 'bullmq'; -import { spawn } from 'child_process'; -import redis from '@lib/redis'; -import prisma from '@lib/prisma'; -import { CrawlStatus } from '@models/crawlSession'; - -export const crawlWorker = new Worker('crawl-tasks', async (job) => { - const { sessionId } = job.data; - // only do it if its status is QUEUED, otherwise it means that the session was aborted while it was in the queue, and we should not start it. - const session = await prisma.crawlSession.findFirstOrThrow({ - where: { id: sessionId } - }); - if (session.status !== CrawlStatus[CrawlStatus.QUEUED]) { - throw new Error(`Cannot start session with status ${session.status}`); - } - - await prisma.crawlSession.update({ - where: { id: sessionId }, - data: { - status: CrawlStatus[CrawlStatus.RUNNING], - startedAt: new Date(), - }, - }); +import { workerRedis } from "@lib/redis"; +import { cacheDel, cacheKeys, cacheSetString } from "@lib/cache"; +import { logger } from "@services/logger.service"; +import { + getCrawlerJobPayload, + markQueuedSessionRunning, + isSessionAborted, + markSessionCompletedIfRunning, + markSessionFailedIfRunning, + markSessionFinishedAtIfAborted, +} from "@services/crawlSession.service"; +import { Worker } from "bullmq"; +import { spawn } from "node:child_process"; +import path from "node:path"; + +function resolveCrawlerWorkdir(): string { + const configured = process.env.CRAWLER_WORKDIR; + if (configured && configured.trim()) return configured; + return path.resolve(process.cwd(), "../coverit-crawler"); +} + +function resolvePythonCommand(): string { + return process.env.CRAWLER_PYTHON?.trim() || "python"; +} + +function resolveCrawlerModule(): string { + return process.env.CRAWLER_MODULE?.trim() || "src.workers.crawler_worker"; +} + +async function runCrawler(sessionId: string): Promise { + const payload = await getCrawlerJobPayload(sessionId); + const python = resolvePythonCommand(); + const cwd = resolveCrawlerWorkdir(); + const moduleName = resolveCrawlerModule(); + + const args: string[] = ["-m", moduleName, "--payload-stdin"]; + + await new Promise(async (resolve, reject) => { + await markQueuedSessionRunning(sessionId); + + const finalizeAbortedIfNeeded = async (): Promise => { + try { + await markSessionFinishedAtIfAborted(sessionId); + } catch (e) { + logger.error(e); + } + }; - const pythonBinary = process.env.CRAWLER_PYTHON_BIN ?? 'python3'; + let child: ReturnType; + try { + child = spawn(python, args, { + cwd, + env: process.env, + stdio: ["pipe", "pipe", "pipe"], + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + try { + const updated = await markSessionFailedIfRunning(sessionId, `Failed to spawn crawler process. ${message}`); + if (!updated) await finalizeAbortedIfNeeded(); + } catch (inner) { + logger.error(inner); + } + return reject(err); + } - const pythonProcess = spawn(pythonBinary, ['main.py', '--session', sessionId], { - stdio: ['pipe', 'pipe', 'pipe'], - cwd: 'src/workers', - detached: false, - }); + const { stdin, stdout, stderr: childStderr } = child; + if (!stdin || !stdout || !childStderr) { + const message = "Crawler process started without stdio pipes."; + try { + const updated = await markSessionFailedIfRunning(sessionId, message); + if (!updated) await finalizeAbortedIfNeeded(); + } catch (e) { + logger.error(e); + } + return reject(new Error(message)); + } - const pid = pythonProcess.pid; + const pid = child.pid; if (pid === undefined) { - throw new Error(`Failed to start crawler process for session ${sessionId}`); + const message = `Failed to start crawler process for session ${sessionId}`; + try { + const updated = await markSessionFailedIfRunning(sessionId, message); + if (!updated) await finalizeAbortedIfNeeded(); + } catch (e) { + logger.error(e); + } + return reject(new Error(message)); } - await redis.set(`session:${sessionId}:pid`, String(pid), 'EX', 86400); - const cleanup = async () => { - await redis.del(`session:${sessionId}:pid`); - }; + const pidKey = cacheKeys.crawlSession.pid(sessionId); + await cacheSetString(pidKey, String(pid), 86400); - pythonProcess.once('close', () => { - void cleanup(); - }); + const cleanup = async (): Promise => { + await cacheDel([pidKey]); + }; - pythonProcess.stdout.on('data', (data) => { - console.log(`[Session ${sessionId}] Python: ${data}`); + child.once("exit", () => { + void cleanup(); }); - pythonProcess.stderr.on('data', (data) => { - console.error(`[Session ${sessionId}] Error: ${data}`); + child.once("error", () => { + void cleanup(); }); - let deleted = false; + const abortInterval = setInterval(() => { + void (async () => { + try { + const aborted = await isSessionAborted(sessionId); + if (!aborted) return; + + try { + child.kill("SIGTERM"); + } catch { + } + setTimeout(() => { + try { + child.kill("SIGKILL"); + } catch { + } + }, 2000); + } catch (e) { + logger.error(e); + } + })(); + }, 1000); - const exitCode = await new Promise((resolve) => { - let lastStatus = CrawlStatus[CrawlStatus.RUNNING]; - let checking = false; + stdin.setDefaultEncoding("utf8"); + stdin.end(JSON.stringify(payload)); - const checkInterval = setInterval(async () => { - if (checking) { - return; - } + let stderr = ""; - checking = true; - let current; - try { - current = await prisma.crawlSession.findFirstOrThrow({ - where: { id: sessionId }, - select: { status: true } - }); - } catch (e) { - // this means that the session was deleted while the crawl was running. In this case, we should stop the crawl immediately. - pythonProcess.stdin.write('ABORT\n'); - clearInterval(checkInterval); - setTimeout(() => pythonProcess.kill('SIGKILL'), 2000); - deleted = true; - return; - } + stdout.setEncoding("utf8"); + childStderr.setEncoding("utf8"); - checking = false; + stdout.on("data", (chunk: string) => { + const lines = chunk.split(/\r?\n/).filter(Boolean); + for (const line of lines) logger.info(line); + }); - if (!current) return; - if (current.status === CrawlStatus[CrawlStatus.ABORTED]) { - pythonProcess.stdin.write('ABORT\n'); - clearInterval(checkInterval); - setTimeout(() => pythonProcess.kill('SIGKILL'), 2000); - return; - } + childStderr.on("data", (chunk: string) => { + stderr += chunk; + if (stderr.length > 64_000) stderr = stderr.slice(stderr.length - 64_000); + const lines = chunk.split(/\r?\n/).filter(Boolean); + for (const line of lines) logger.error(line); + }); - if (current.status !== lastStatus) { - if (current.status === CrawlStatus[CrawlStatus.PAUSED]) { - pythonProcess.stdin.write('PAUSE\n'); - } else if (current.status === CrawlStatus[CrawlStatus.RUNNING]) { - pythonProcess.stdin.write('RESUME\n'); - } - lastStatus = current.status; - } - }, 3000); - - pythonProcess.on('close', (code) => { - clearInterval(checkInterval); - resolve(code); - }); - - pythonProcess.on('error', (err) => { - console.error("Failed to start child process:", err); - clearInterval(checkInterval); - resolve(1); - }); + child.on("error", (err) => { + clearInterval(abortInterval); + void (async () => { + try { + const updated = await markSessionFailedIfRunning(sessionId, `Crawler process error. ${err.message}`); + if (!updated) await finalizeAbortedIfNeeded(); + } catch (inner) { + logger.error(inner); + } + })(); + reject(err); }); - if (!deleted) { - const finalSession = await prisma.crawlSession.findUnique({ where: { id: sessionId } }); - const isAborted = finalSession?.status === CrawlStatus[CrawlStatus.ABORTED]; - const isStopped = finalSession?.status === CrawlStatus[CrawlStatus.PAUSED]; - - if (exitCode === 0) { - if (!isAborted && !isStopped) { - await prisma.crawlSession.update({ - where: { id: sessionId }, - data: { status: CrawlStatus[CrawlStatus.COMPLETED], finishedAt: new Date() } - }); - } - } else { - if (!isAborted) { - await prisma.crawlSession.update({ - where: { id: sessionId }, - data: { status: CrawlStatus[CrawlStatus.FAILED], finishedAt: new Date() } - }); - } + child.on("exit", (code) => { + clearInterval(abortInterval); + void (async () => { + if (code === 0) { + try { + const updated = await markSessionCompletedIfRunning(sessionId); + if (!updated) await finalizeAbortedIfNeeded(); + } catch (err) { + logger.error(err); + } + return resolve(); } - } -}, { connection: redis }); \ No newline at end of file + + const message = `Crawler process exited with code ${code}. ${stderr.trim()}`.trim(); + try { + const updated = await markSessionFailedIfRunning(sessionId, message); + if (!updated) await finalizeAbortedIfNeeded(); + } catch (err) { + logger.error(err); + } + reject(new Error(message)); + })(); + }); + }); +} + +new Worker( + "crawl-tasks", + async (job) => { + if (job.name !== "crawl") return; + await runCrawler(job.data.sessionId); + }, + { connection: workerRedis }, +); + +logger.info("[Worker] Crawler worker started and listening for jobs..."); \ No newline at end of file From 22952dc1e32028df2d044d33c071a3289f43f654 Mon Sep 17 00:00:00 2001 From: Youssef Date: Sat, 11 Apr 2026 20:23:18 +0200 Subject: [PATCH 3/6] refactor: put type in types folder --- package-lock.json | 1193 +++++++++++++++++---------------- src/types/crawler.ts | 2 +- src/workers/crawler.worker.ts | 4 + 3 files changed, 631 insertions(+), 568 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4cdb940..958f9a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -108,9 +108,9 @@ } }, "node_modules/@asteasolutions/zod-to-openapi": { - "version": "8.4.3", - "resolved": "https://registry.npmjs.org/@asteasolutions/zod-to-openapi/-/zod-to-openapi-8.4.3.tgz", - "integrity": "sha512-lwfMTN7kDbFDwMniYZUebiGGHxVGBw9ZSI4IBYjm6Ey22Kd5z/fsQb2k+Okr8WMbCCC553vi/ZM9utl5/XcvuQ==", + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/@asteasolutions/zod-to-openapi/-/zod-to-openapi-8.5.0.tgz", + "integrity": "sha512-SABbKiObg5dLRiTFnqiW1WWwGcg1BJfmHtT2asIBnBHg6Smy/Ms2KHc650+JI4Hw7lSkdiNebEGXpwoxfben8Q==", "license": "MIT", "dependencies": { "openapi3-ts": "^4.1.2" @@ -312,23 +312,23 @@ } }, "node_modules/@babel/helpers": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.6.tgz", - "integrity": "sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==", + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.2.tgz", + "integrity": "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==", "dev": true, "license": "MIT", "dependencies": { "@babel/template": "^7.28.6", - "@babel/types": "^7.28.6" + "@babel/types": "^7.29.0" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.0.tgz", - "integrity": "sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==", + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", + "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", "dev": true, "license": "MIT", "dependencies": { @@ -641,51 +641,18 @@ "integrity": "sha512-sBXGT13cpmPR5BMgHE6UEEfEaShh5Ror6rfN3yEK5si7QVrtZg8LEPQb0VVhiLRUslD2yLnXtnRzG035J/mZXQ==", "license": "(Apache-2.0 AND BSD-3-Clause)" }, - "node_modules/@chevrotain/cst-dts-gen": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/@chevrotain/cst-dts-gen/-/cst-dts-gen-10.5.0.tgz", - "integrity": "sha512-lhmC/FyqQ2o7pGK4Om+hzuDrm9rhFYIJ/AXoQBeongmn870Xeb0L6oGEiuR8nohFNL5sMaQEJWCxr1oIVIVXrw==", - "license": "Apache-2.0", - "dependencies": { - "@chevrotain/gast": "10.5.0", - "@chevrotain/types": "10.5.0", - "lodash": "4.17.21" - } - }, - "node_modules/@chevrotain/gast": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/@chevrotain/gast/-/gast-10.5.0.tgz", - "integrity": "sha512-pXdMJ9XeDAbgOWKuD1Fldz4ieCs6+nLNmyVhe2gZVqoO7v8HXuHYs5OV2EzUtbuai37TlOAQHrTDvxMnvMJz3A==", - "license": "Apache-2.0", - "dependencies": { - "@chevrotain/types": "10.5.0", - "lodash": "4.17.21" - } - }, - "node_modules/@chevrotain/types": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/@chevrotain/types/-/types-10.5.0.tgz", - "integrity": "sha512-f1MAia0x/pAVPWH/T73BJVyO2XU5tI4/iE7cnxb7tqdNTNhQI3Uq3XkqcoteTmD4t1aM0LbHCJOhgIDn07kl2A==", - "license": "Apache-2.0" - }, - "node_modules/@chevrotain/utils": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/@chevrotain/utils/-/utils-10.5.0.tgz", - "integrity": "sha512-hBzuU5+JjB2cqNZyszkDHZgOSrUUT8V3dhgRl8Q9Gp6dAj/H5+KILGjbhDpc3Iy9qmqlm/akuOI2ut9VUtzJxQ==", - "license": "Apache-2.0" - }, "node_modules/@commitlint/cli": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-20.4.4.tgz", - "integrity": "sha512-GLMNQHYGcn0ohL2HMlAnXcD1PS2vqBBGbYKlhrRPOYsWiRoLWtrewsR3uKRb9v/IdS+qOS0vqJQ64n1g8VPKFw==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-20.5.0.tgz", + "integrity": "sha512-yNkyN/tuKTJS3wdVfsZ2tXDM4G4Gi7z+jW54Cki8N8tZqwKBltbIvUUrSbT4hz1bhW/h0CdR+5sCSpXD+wMKaQ==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/format": "^20.4.4", - "@commitlint/lint": "^20.4.4", - "@commitlint/load": "^20.4.4", - "@commitlint/read": "^20.4.4", - "@commitlint/types": "^20.4.4", + "@commitlint/format": "^20.5.0", + "@commitlint/lint": "^20.5.0", + "@commitlint/load": "^20.5.0", + "@commitlint/read": "^20.5.0", + "@commitlint/types": "^20.5.0", "tinyexec": "^1.0.0", "yargs": "^17.0.0" }, @@ -697,13 +664,13 @@ } }, "node_modules/@commitlint/config-conventional": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-20.4.4.tgz", - "integrity": "sha512-Usg+XXbPNG2GtFWTgRURNWCge1iH1y6jQIvvklOdAbyn2t8ajfVwZCnf5t5X4gUsy17BOiY+myszGsSMIvhOVA==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-20.5.0.tgz", + "integrity": "sha512-t3Ni88rFw1XMa4nZHgOKJ8fIAT9M2j5TnKyTqJzsxea7FUetlNdYFus9dz+MhIRZmc16P0PPyEfh6X2d/qw8SA==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/types": "^20.4.4", + "@commitlint/types": "^20.5.0", "conventional-changelog-conventionalcommits": "^9.2.0" }, "engines": { @@ -711,13 +678,13 @@ } }, "node_modules/@commitlint/config-validator": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-20.4.4.tgz", - "integrity": "sha512-K8hMS9PTLl7EYe5vWtSFQ/sgsV2PHUOtEnosg8k3ZQxCyfKD34I4C7FxWEfRTR54rFKeUYmM3pmRQqBNQeLdlw==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-20.5.0.tgz", + "integrity": "sha512-T/Uh6iJUzyx7j35GmHWdIiGRQB+ouZDk0pwAaYq4SXgB54KZhFdJ0vYmxiW6AMYICTIWuyMxDBl1jK74oFp/Gw==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/types": "^20.4.4", + "@commitlint/types": "^20.5.0", "ajv": "^8.11.0" }, "engines": { @@ -725,15 +692,15 @@ } }, "node_modules/@commitlint/cz-commitlint": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/cz-commitlint/-/cz-commitlint-20.4.4.tgz", - "integrity": "sha512-dQuLSHrLbeLx/7JI0bCeYI2sWmmEs8rCUwIZu0d6p9C4OjnjPcLxQOBRaFwiqXGrjr1Dctb2dIzNtymt06vTQA==", + "version": "20.5.1", + "resolved": "https://registry.npmjs.org/@commitlint/cz-commitlint/-/cz-commitlint-20.5.1.tgz", + "integrity": "sha512-tvcHpk/DQRF749PuaYKPeLhml4grD4h8qEIW5uoboKy0tfnJxmAy99xOo2U1xpQj5uo3LH7pjnVv1xvuhzLWRw==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/ensure": "^20.4.4", - "@commitlint/load": "^20.4.4", - "@commitlint/types": "^20.4.4", + "@commitlint/ensure": "^20.5.0", + "@commitlint/load": "^20.5.0", + "@commitlint/types": "^20.5.0", "is-plain-obj": "^4.1.0", "picocolors": "^1.1.1", "word-wrap": "^1.2.5" @@ -747,13 +714,13 @@ } }, "node_modules/@commitlint/ensure": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/ensure/-/ensure-20.4.4.tgz", - "integrity": "sha512-QivV0M1MGL867XCaF+jJkbVXEPKBALhUUXdjae66hes95aY1p3vBJdrcl3x8jDv2pdKWvIYIz+7DFRV/v0dRkA==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/ensure/-/ensure-20.5.0.tgz", + "integrity": "sha512-IpHqAUesBeW1EDDdjzJeaOxU9tnogLAyXLRBn03SHlj1SGENn2JGZqSWGkFvBJkJzfXAuCNtsoYzax+ZPS+puw==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/types": "^20.4.4", + "@commitlint/types": "^20.5.0", "lodash.camelcase": "^4.3.0", "lodash.kebabcase": "^4.1.1", "lodash.snakecase": "^4.1.1", @@ -775,13 +742,13 @@ } }, "node_modules/@commitlint/format": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/format/-/format-20.4.4.tgz", - "integrity": "sha512-jLi/JBA4GEQxc5135VYCnkShcm1/rarbXMn2Tlt3Si7DHiiNKHm4TaiJCLnGbZ1r8UfwDRk+qrzZ80kwh08Aow==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/format/-/format-20.5.0.tgz", + "integrity": "sha512-TI9EwFU/qZWSK7a5qyXMpKPPv3qta7FO4tKW+Wt2al7sgMbLWTsAcDpX1cU8k16TRdsiiet9aOw0zpvRXNJu7Q==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/types": "^20.4.4", + "@commitlint/types": "^20.5.0", "picocolors": "^1.1.1" }, "engines": { @@ -789,13 +756,13 @@ } }, "node_modules/@commitlint/is-ignored": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-20.4.4.tgz", - "integrity": "sha512-y76rT8yq02x+pMDBI2vY4y/ByAwmJTkta/pASbgo8tldBiKLduX8/2NCRTSCjb3SumE5FBeopERKx3oMIm8RTQ==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-20.5.0.tgz", + "integrity": "sha512-JWLarAsurHJhPozbuAH6GbP4p/hdOCoqS9zJMfqwswne+/GPs5V0+rrsfOkP68Y8PSLphwtFXV0EzJ+GTXTTGg==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/types": "^20.4.4", + "@commitlint/types": "^20.5.0", "semver": "^7.6.0" }, "engines": { @@ -803,32 +770,32 @@ } }, "node_modules/@commitlint/lint": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-20.4.4.tgz", - "integrity": "sha512-svOEW+RptcNpXKE7UllcAsV0HDIdOck9reC2TP1QA6K5Fo0xxQV+QPjV8Zqx9g6X/hQBkF2S9ZQZ78Xrv1Eiog==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-20.5.0.tgz", + "integrity": "sha512-jiM3hNUdu04jFBf1VgPdjtIPvbuVfDTBAc6L98AWcoLjF5sYqkulBHBzlVWll4rMF1T5zeQFB6r//a+s+BBKlA==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/is-ignored": "^20.4.4", - "@commitlint/parse": "^20.4.4", - "@commitlint/rules": "^20.4.4", - "@commitlint/types": "^20.4.4" + "@commitlint/is-ignored": "^20.5.0", + "@commitlint/parse": "^20.5.0", + "@commitlint/rules": "^20.5.0", + "@commitlint/types": "^20.5.0" }, "engines": { "node": ">=v18" } }, "node_modules/@commitlint/load": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-20.4.4.tgz", - "integrity": "sha512-kvFrzvoIACa/fMjXEP0LNEJB1joaH3q3oeMJsLajXE5IXjYrNGVcW1ZFojXUruVJ7odTZbC3LdE/6+ONW4f2Dg==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-20.5.0.tgz", + "integrity": "sha512-sLhhYTL/KxeOTZjjabKDhwidGZan84XKK1+XFkwDYL/4883kIajcz/dZFAhBJmZPtL8+nBx6bnkzA95YxPeDPw==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/config-validator": "^20.4.4", + "@commitlint/config-validator": "^20.5.0", "@commitlint/execute-rule": "^20.0.0", - "@commitlint/resolve-extends": "^20.4.4", - "@commitlint/types": "^20.4.4", + "@commitlint/resolve-extends": "^20.5.0", + "@commitlint/types": "^20.5.0", "cosmiconfig": "^9.0.1", "cosmiconfig-typescript-loader": "^6.1.0", "is-plain-obj": "^4.1.0", @@ -850,13 +817,13 @@ } }, "node_modules/@commitlint/parse": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/parse/-/parse-20.4.4.tgz", - "integrity": "sha512-AjfgOgrjEozeQNzhFu1KL5N0nDx4JZmswVJKNfOTLTUGp6xODhZHCHqb//QUHKOzx36If5DQ7tci2o7szYxu1A==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/parse/-/parse-20.5.0.tgz", + "integrity": "sha512-SeKWHBMk7YOTnnEWUhx+d1a9vHsjjuo6Uo1xRfPNfeY4bdYFasCH1dDpAv13Lyn+dDPOels+jP6D2GRZqzc5fA==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/types": "^20.4.4", + "@commitlint/types": "^20.5.0", "conventional-changelog-angular": "^8.2.0", "conventional-commits-parser": "^6.3.0" }, @@ -865,14 +832,14 @@ } }, "node_modules/@commitlint/read": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-20.4.4.tgz", - "integrity": "sha512-jvgdAQDdEY6L8kCxOo21IWoiAyNFzvrZb121wU2eBxI1DzWAUZgAq+a8LlJRbT0Qsj9INhIPVWgdaBbEzlF0dQ==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-20.5.0.tgz", + "integrity": "sha512-JDEIJ2+GnWpK8QqwfmW7O42h0aycJEWNqcdkJnyzLD11nf9dW2dWLTVEa8Wtlo4IZFGLPATjR5neA5QlOvIH1w==", "dev": true, "license": "MIT", "dependencies": { "@commitlint/top-level": "^20.4.3", - "@commitlint/types": "^20.4.4", + "@commitlint/types": "^20.5.0", "git-raw-commits": "^5.0.0", "minimist": "^1.2.8", "tinyexec": "^1.0.0" @@ -882,14 +849,14 @@ } }, "node_modules/@commitlint/resolve-extends": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-20.4.4.tgz", - "integrity": "sha512-pyOf+yX3c3m/IWAn2Jop+7s0YGKPQ8YvQaxt9IQxnLIM3yZAlBdkKiQCT14TnrmZTkVGTXiLtckcnFTXYwlY0A==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-20.5.0.tgz", + "integrity": "sha512-3SHPWUW2v0tyspCTcfSsYml0gses92l6TlogwzvM2cbxDgmhSRc+fldDjvGkCXJrjSM87BBaWYTPWwwyASZRrg==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/config-validator": "^20.4.4", - "@commitlint/types": "^20.4.4", + "@commitlint/config-validator": "^20.5.0", + "@commitlint/types": "^20.5.0", "global-directory": "^4.0.1", "import-meta-resolve": "^4.0.0", "lodash.mergewith": "^4.6.2", @@ -900,16 +867,16 @@ } }, "node_modules/@commitlint/rules": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/rules/-/rules-20.4.4.tgz", - "integrity": "sha512-PmUp8QPLICn9w05dAx5r1rdOYoTk7SkfusJJh5tP3TqHwo2mlQ9jsOm8F0HSXU9kuLfgTEGNrunAx/dlK/RyPQ==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/rules/-/rules-20.5.0.tgz", + "integrity": "sha512-5NdQXQEdnDPT5pK8O39ZA7HohzPRHEsDGU23cyVCNPQy4WegAbAwrQk3nIu7p2sl3dutPk8RZd91yKTrMTnRkQ==", "dev": true, "license": "MIT", "dependencies": { - "@commitlint/ensure": "^20.4.4", + "@commitlint/ensure": "^20.5.0", "@commitlint/message": "^20.4.3", "@commitlint/to-lines": "^20.0.0", - "@commitlint/types": "^20.4.4" + "@commitlint/types": "^20.5.0" }, "engines": { "node": ">=v18" @@ -939,9 +906,9 @@ } }, "node_modules/@commitlint/types": { - "version": "20.4.4", - "resolved": "https://registry.npmjs.org/@commitlint/types/-/types-20.4.4.tgz", - "integrity": "sha512-dwTGzyAblFXHJNBOgrTuO5Ee48ioXpS5XPRLLatxhQu149DFAHUcB3f0Q5eea3RM4USSsP1+WVT2dBtLVod4fg==", + "version": "20.5.0", + "resolved": "https://registry.npmjs.org/@commitlint/types/-/types-20.5.0.tgz", + "integrity": "sha512-ZJoS8oSq2CAZEpc/YI9SulLrdiIyXeHb/OGqGrkUP6Q7YV+0ouNAa7GjqRdXeQPncHQIDz/jbCTlHScvYvO/gA==", "dev": true, "license": "MIT", "dependencies": { @@ -953,9 +920,9 @@ } }, "node_modules/@conventional-changelog/git-client": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@conventional-changelog/git-client/-/git-client-2.6.0.tgz", - "integrity": "sha512-T+uPDciKf0/ioNNDpMGc8FDsehJClZP0yR3Q5MN6wE/Y/1QZ7F+80OgznnTCOlMEG4AV0LvH2UJi3C/nBnaBUg==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@conventional-changelog/git-client/-/git-client-2.7.0.tgz", + "integrity": "sha512-j7A8/LBEQ+3rugMzPXoKYzyUPpw/0CBQCyvtTR7Lmu4olG4yRC/Tfkq79Mr3yuPs0SUitlO2HwGP3gitMJnRFw==", "dev": true, "license": "MIT", "dependencies": { @@ -968,7 +935,7 @@ }, "peerDependencies": { "conventional-commits-filter": "^5.0.0", - "conventional-commits-parser": "^6.3.0" + "conventional-commits-parser": "^6.4.0" }, "peerDependenciesMeta": { "conventional-commits-filter": { @@ -989,7 +956,7 @@ }, "node_modules/@coveritlabs/git-hooks": { "version": "1.0.0", - "resolved": "git+ssh://git@github.com/CoveritLabs/.github.git#8b96a0dc12c3084dfa9db3085840ff633965a167", + "resolved": "git+ssh://git@github.com/CoveritLabs/.github.git#a48c212fecda8290a0715c4a4c4632fb9b8a692f", "dev": true, "license": "UNLICENSED", "dependencies": { @@ -1035,42 +1002,42 @@ "license": "Apache-2.0" }, "node_modules/@electric-sql/pglite-socket": { - "version": "0.0.20", - "resolved": "https://registry.npmjs.org/@electric-sql/pglite-socket/-/pglite-socket-0.0.20.tgz", - "integrity": "sha512-J5nLGsicnD9wJHnno9r+DGxfcZWh+YJMCe0q/aCgtG6XOm9Z7fKeite8IZSNXgZeGltSigM9U/vAWZQWdgcSFg==", + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite-socket/-/pglite-socket-0.1.1.tgz", + "integrity": "sha512-p2hoXw3Z3LQHwTeikdZNsFBOvXGqKY2hk51BBw+8NKND8eoH+8LFOtW9Z8CQKmTJ2qqGYu82ipqiyFZOTTXNfw==", "license": "Apache-2.0", "bin": { "pglite-server": "dist/scripts/server.js" }, "peerDependencies": { - "@electric-sql/pglite": "0.3.15" + "@electric-sql/pglite": "0.4.1" } }, "node_modules/@electric-sql/pglite-tools": { - "version": "0.2.20", - "resolved": "https://registry.npmjs.org/@electric-sql/pglite-tools/-/pglite-tools-0.2.20.tgz", - "integrity": "sha512-BK50ZnYa3IG7ztXhtgYf0Q7zijV32Iw1cYS8C+ThdQlwx12V5VZ9KRJ42y82Hyb4PkTxZQklVQA9JHyUlex33A==", + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite-tools/-/pglite-tools-0.3.1.tgz", + "integrity": "sha512-C+T3oivmy9bpQvSxVqXA1UDY8cB9Eb9vZHL9zxWwEUfDixbXv4G3r2LjoTdR33LD8aomR3O9ZXEO3XEwr/cUCA==", "license": "Apache-2.0", "peerDependencies": { - "@electric-sql/pglite": "0.3.15" + "@electric-sql/pglite": "0.4.1" } }, "node_modules/@emnapi/core": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.0.tgz", - "integrity": "sha512-0DQ98G9ZQZOxfUcQn1waV2yS8aWdZ6kJMbYCJB3oUBecjWYO1fqJ+a1DRfPF3O5JEkwqwP1A9QEN/9mYm2Yd0w==", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz", + "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.2.0", + "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.0.tgz", - "integrity": "sha512-QN75eB0IH2ywSpRpNddCRfQIhmJYBCJ1x5Lb3IscKAL8bMnVAKnRg8dCoXbHzVLLH7P38N2Z3mtulB7W0J0FKw==", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", + "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", "dev": true, "license": "MIT", "optional": true, @@ -1079,9 +1046,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.0.tgz", - "integrity": "sha512-N10dEJNSsUx41Z6pZsXU8FjPjpBEplgH24sfkmITrBED1/U2Esum9F3lfLrMjKHHjmi557zQn7kR9R+XWXu5Rg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", "dev": true, "license": "MIT", "optional": true, @@ -1277,9 +1244,9 @@ } }, "node_modules/@hono/node-server": { - "version": "1.19.9", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.9.tgz", - "integrity": "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw==", + "version": "1.19.11", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.11.tgz", + "integrity": "sha512-dr8/3zEaB+p0D2n/IUrlPF1HZm586qgJNXK1a9fhg/PzdtkK7Ksd5l312tJX2yBuALqDYBlG20QEbayqPyxn+g==", "license": "MIT", "engines": { "node": ">=18.14.1" @@ -1996,9 +1963,9 @@ } }, "node_modules/@jest/reporters/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", "dev": true, "license": "MIT", "dependencies": { @@ -2508,18 +2475,11 @@ "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==", "license": "MIT" }, - "node_modules/@mrleebo/prisma-ast": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/@mrleebo/prisma-ast/-/prisma-ast-0.13.1.tgz", - "integrity": "sha512-XyroGQXcHrZdvmrGJvsA9KNeOOgGMg1Vg9OlheUsBOSKznLMDl+YChxbkboRHvtFYJEMRYmlV3uoo/njCw05iw==", - "license": "MIT", - "dependencies": { - "chevrotain": "^10.5.0", - "lilconfig": "^2.1.0" - }, - "engines": { - "node": ">=16" - } + "node_modules/@kurkle/color": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.4.tgz", + "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==", + "license": "MIT" }, "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { "version": "3.0.3", @@ -2713,83 +2673,24 @@ } }, "node_modules/@prisma/adapter-pg": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@prisma/adapter-pg/-/adapter-pg-7.5.0.tgz", - "integrity": "sha512-EJx7OLULahcC3IjJgdx2qRDNCT+ToY2v66UkeETMCLhNOTgqVzRzYvOEphY7Zp0eHyzfkC33Edd/qqeadf9R4A==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@prisma/adapter-pg/-/adapter-pg-7.7.0.tgz", + "integrity": "sha512-q33Ta8sKbgzEpAy0lx45tAq//yMv0qcb+8nj+TCA3P4wiAY+OBFEFk/NDkZncAfHaNJeGo5WJpJdpbL+ijYx8g==", "license": "Apache-2.0", "dependencies": { - "@prisma/driver-adapter-utils": "7.5.0", - "@types/pg": "8.11.11", + "@prisma/driver-adapter-utils": "7.7.0", + "@types/pg": "^8.16.0", "pg": "^8.16.3", "postgres-array": "3.0.4" } }, - "node_modules/@prisma/adapter-pg/node_modules/@types/pg": { - "version": "8.11.11", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.11.11.tgz", - "integrity": "sha512-kGT1qKM8wJQ5qlawUrEkXgvMSXoV213KfMGXcwfDwUIfUHXqXYXOfS1nE1LINRJVVVx5wCm70XnFlMHaIcQAfw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^4.0.1" - } - }, - "node_modules/@prisma/adapter-pg/node_modules/pg-types": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-4.1.0.tgz", - "integrity": "sha512-o2XFanIMy/3+mThw69O8d4n1E5zsLhdO+OPqswezu7Z5ekP4hYDqlDjlmOpYMbzY2Br0ufCwJLdDIXeNVwcWFg==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "pg-numeric": "1.0.2", - "postgres-array": "~3.0.1", - "postgres-bytea": "~3.0.0", - "postgres-date": "~2.1.0", - "postgres-interval": "^3.0.0", - "postgres-range": "^1.1.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@prisma/adapter-pg/node_modules/postgres-bytea": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-3.0.0.tgz", - "integrity": "sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==", - "license": "MIT", - "dependencies": { - "obuf": "~1.1.2" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/@prisma/adapter-pg/node_modules/postgres-date": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-2.1.0.tgz", - "integrity": "sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==", - "license": "MIT", - "engines": { - "node": ">=12" - } - }, - "node_modules/@prisma/adapter-pg/node_modules/postgres-interval": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-3.0.0.tgz", - "integrity": "sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==", - "license": "MIT", - "engines": { - "node": ">=12" - } - }, "node_modules/@prisma/client": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@prisma/client/-/client-7.5.0.tgz", - "integrity": "sha512-h4hF9ctp+kSRs7ENHGsFQmHAgHcfkOCxbYt6Ti9Xi8x7D+kP4tTi9x51UKmiTH/OqdyJAO+8V+r+JA5AWdav7w==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-7.7.0.tgz", + "integrity": "sha512-5Ar4OsZpJ54s21sy5oDNNW9gQtd4NuxCaiM7+JDTOU07D6VvlpLjYzAVCMB1+JzokN+08dAVomlx+b7bhJd3ww==", "license": "Apache-2.0", "dependencies": { - "@prisma/client-runtime-utils": "7.5.0" + "@prisma/client-runtime-utils": "7.7.0" }, "engines": { "node": "^20.19 || ^22.12 || >=24.0" @@ -2808,45 +2709,45 @@ } }, "node_modules/@prisma/client-runtime-utils": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@prisma/client-runtime-utils/-/client-runtime-utils-7.5.0.tgz", - "integrity": "sha512-KnJ2b4Si/pcWEtK68uM+h0h1oh80CZt2suhLTVuLaSKg4n58Q9jBF/A42Kw6Ma+aThy1yAhfDeTC0JvEmeZnFQ==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@prisma/client-runtime-utils/-/client-runtime-utils-7.7.0.tgz", + "integrity": "sha512-BLyd0UpFYOtyJFTHm7jS9vesHW7P83abibodQMiIofqjBKzDHQ1VAsQkdfvXyYDkPlONPfOTz7/rv3x/+CQqvQ==", "license": "Apache-2.0" }, "node_modules/@prisma/config": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@prisma/config/-/config-7.5.0.tgz", - "integrity": "sha512-1J/9YEX7A889xM46PYg9e8VAuSL1IUmXJW3tEhMv7XQHDWlfC9YSkIw9sTYRaq5GswGlxZ+GnnyiNsUZ9JJhSQ==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@prisma/config/-/config-7.7.0.tgz", + "integrity": "sha512-hmPI3tKLO2aP0Y5vugbjcnA9qqlfJndiT6ds4tw28U5hNHLWg+mHJEWAhjsSPgxjtmxhJ/EDIeIlyh+3Us0OPg==", "license": "Apache-2.0", "dependencies": { "c12": "3.1.0", "deepmerge-ts": "7.1.5", - "effect": "3.18.4", + "effect": "3.20.0", "empathic": "2.0.0" } }, "node_modules/@prisma/debug": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-7.5.0.tgz", - "integrity": "sha512-163+nffny0JoPEkDhfNco0vcuT3ymIJc9+WX7MHSQhfkeKUmKe9/wqvGk5SjppT93DtBjVwr5HPJYlXbzm6qtg==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-7.7.0.tgz", + "integrity": "sha512-12J62XdqCmpiwJHhHdQxZeY3ckVCWIFmcJP8hg5dPTceeiQ0wiojXGFYTluKqFQfu46fRLgb/rLALZMAx3+dTA==", "license": "Apache-2.0" }, "node_modules/@prisma/dev": { - "version": "0.20.0", - "resolved": "https://registry.npmjs.org/@prisma/dev/-/dev-0.20.0.tgz", - "integrity": "sha512-ovlBYwWor0OzG+yH4J3Ot+AneD818BttLA+Ii7wjbcLHUrnC4tbUPVGyNd3c/+71KETPKZfjhkTSpdS15dmXNQ==", + "version": "0.24.3", + "resolved": "https://registry.npmjs.org/@prisma/dev/-/dev-0.24.3.tgz", + "integrity": "sha512-ffHlQuKXZiaDt9Go0OnCTdJZrHxK0k7omJKNV86/VjpsXu5EIHZLK0T7JSWgvNlJwh56kW9JFu9v0qJciFzepg==", "license": "ISC", "dependencies": { - "@electric-sql/pglite": "0.3.15", - "@electric-sql/pglite-socket": "0.0.20", - "@electric-sql/pglite-tools": "0.2.20", - "@hono/node-server": "1.19.9", - "@mrleebo/prisma-ast": "0.13.1", + "@electric-sql/pglite": "0.4.1", + "@electric-sql/pglite-socket": "0.1.1", + "@electric-sql/pglite-tools": "0.3.1", + "@hono/node-server": "1.19.11", "@prisma/get-platform": "7.2.0", "@prisma/query-plan-executor": "7.2.0", + "@prisma/streams-local": "0.1.2", "foreground-child": "3.3.1", "get-port-please": "3.2.0", - "hono": "4.11.4", + "hono": "^4.12.8", "http-status-codes": "2.3.0", "pathe": "2.0.3", "proper-lockfile": "4.1.2", @@ -2857,60 +2758,60 @@ } }, "node_modules/@prisma/driver-adapter-utils": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@prisma/driver-adapter-utils/-/driver-adapter-utils-7.5.0.tgz", - "integrity": "sha512-B79N/amgV677mFesFDBAdrW0OIaqawap9E0sjgLBtzIz2R3hIMS1QB8mLZuUEiS4q5Y8Oh3I25Kw4SLxMypk9Q==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@prisma/driver-adapter-utils/-/driver-adapter-utils-7.7.0.tgz", + "integrity": "sha512-gZXREeu6mOk7zXfGFJgh86p7Vhj0sXNKp+4Cg1tWYo7V2dfncP2qxS2BiTmbIIha8xPqItkl0WSw38RuSq1HoQ==", "license": "Apache-2.0", "dependencies": { - "@prisma/debug": "7.5.0" + "@prisma/debug": "7.7.0" } }, "node_modules/@prisma/engines": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-7.5.0.tgz", - "integrity": "sha512-ondGRhzoaVpRWvFaQ5wH5zS1BIbhzbKqczKjCn6j3L0Zfe/LInjcEg8+xtB49AuZBX30qyx1ZtGoootUohz2pw==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-7.7.0.tgz", + "integrity": "sha512-7fmcbT7HHXBq/b+3h/dO1JI3fd8l8q7erf7xP7pRprh58hmSSnG8mg9K3yjW3h9WaHWUwngVFpSxxxivaitQ2w==", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { - "@prisma/debug": "7.5.0", - "@prisma/engines-version": "7.5.0-15.280c870be64f457428992c43c1f6d557fab6e29e", - "@prisma/fetch-engine": "7.5.0", - "@prisma/get-platform": "7.5.0" + "@prisma/debug": "7.7.0", + "@prisma/engines-version": "7.6.0-1.75cbdc1eb7150937890ad5465d861175c6624711", + "@prisma/fetch-engine": "7.7.0", + "@prisma/get-platform": "7.7.0" } }, "node_modules/@prisma/engines-version": { - "version": "7.5.0-15.280c870be64f457428992c43c1f6d557fab6e29e", - "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-7.5.0-15.280c870be64f457428992c43c1f6d557fab6e29e.tgz", - "integrity": "sha512-E+iRV/vbJLl8iGjVr6g/TEWokA+gjkV/doZkaQN1i/ULVdDwGnPJDfLUIFGS3BVwlG/m6L8T4x1x5isl8hGMxA==", + "version": "7.6.0-1.75cbdc1eb7150937890ad5465d861175c6624711", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-7.6.0-1.75cbdc1eb7150937890ad5465d861175c6624711.tgz", + "integrity": "sha512-r51DLcJ8bDRSrBEJF3J4cinoWyGA7rfP2mG6lD90VqIbGNOkbfcLcXalSVjq5Y6brQS3vcjrq4GbyUb1Cb7vkw==", "license": "Apache-2.0" }, "node_modules/@prisma/engines/node_modules/@prisma/get-platform": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-7.5.0.tgz", - "integrity": "sha512-7I+2y1nu/gkEKSiHHbcZ1HPe/euGdEqJZxEEMT0246q4De1+hla0ZzlTgvaT9dHcVCgLSuCG8v39db5qUUWNgw==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-7.7.0.tgz", + "integrity": "sha512-MEUNzvKxvYnJ7kgvd6oNRnMmmiGNS9TYLB2weMeIXplnHdL/UWEGnvavYGnN7KLJ2n0iI4dDAyzSkHI3c7AscQ==", "license": "Apache-2.0", "dependencies": { - "@prisma/debug": "7.5.0" + "@prisma/debug": "7.7.0" } }, "node_modules/@prisma/fetch-engine": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-7.5.0.tgz", - "integrity": "sha512-kZCl2FV54qnyrVdnII8MI6qvt7HfU6Cbiz8dZ8PXz4f4lbSw45jEB9/gEMK2SGdiNhBKyk/Wv95uthoLhGMLYA==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-7.7.0.tgz", + "integrity": "sha512-TfyzveBQoK4xALzsTpVhB/0KG1N8zOK0ap+RnBMkzGUu3f98fnQ4QtXa2wlKPhsO2X8a3N5ugFQgcKNoHGmDfw==", "license": "Apache-2.0", "dependencies": { - "@prisma/debug": "7.5.0", - "@prisma/engines-version": "7.5.0-15.280c870be64f457428992c43c1f6d557fab6e29e", - "@prisma/get-platform": "7.5.0" + "@prisma/debug": "7.7.0", + "@prisma/engines-version": "7.6.0-1.75cbdc1eb7150937890ad5465d861175c6624711", + "@prisma/get-platform": "7.7.0" } }, "node_modules/@prisma/fetch-engine/node_modules/@prisma/get-platform": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-7.5.0.tgz", - "integrity": "sha512-7I+2y1nu/gkEKSiHHbcZ1HPe/euGdEqJZxEEMT0246q4De1+hla0ZzlTgvaT9dHcVCgLSuCG8v39db5qUUWNgw==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-7.7.0.tgz", + "integrity": "sha512-MEUNzvKxvYnJ7kgvd6oNRnMmmiGNS9TYLB2weMeIXplnHdL/UWEGnvavYGnN7KLJ2n0iI4dDAyzSkHI3c7AscQ==", "license": "Apache-2.0", "dependencies": { - "@prisma/debug": "7.5.0" + "@prisma/debug": "7.7.0" } }, "node_modules/@prisma/get-platform": { @@ -2934,13 +2835,45 @@ "integrity": "sha512-EOZmNzcV8uJ0mae3DhTsiHgoNCuu1J9mULQpGCh62zN3PxPTd+qI9tJvk5jOst8WHKQNwJWR3b39t0XvfBB0WQ==", "license": "Apache-2.0" }, + "node_modules/@prisma/streams-local": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@prisma/streams-local/-/streams-local-0.1.2.tgz", + "integrity": "sha512-l49yTxKKF2odFxaAXTmwmkBKL3+bVQ1tFOooGifu4xkdb9NMNLxHj27XAhTylWZod8I+ISGM5erU1xcl/oBCtg==", + "license": "Apache-2.0", + "dependencies": { + "ajv": "^8.12.0", + "better-result": "^2.7.0", + "env-paths": "^3.0.0", + "proper-lockfile": "^4.1.2" + }, + "engines": { + "bun": ">=1.3.6", + "node": ">=22.0.0" + } + }, + "node_modules/@prisma/streams-local/node_modules/env-paths": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", + "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@prisma/studio-core": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/@prisma/studio-core/-/studio-core-0.21.1.tgz", - "integrity": "sha512-bOGqG/eMQtKC0XVvcVLRmhWWzm/I+0QUWqAEhEBtetpuS3k3V4IWqKGUONkAIT223DNXJMxMtZp36b1FmcdPeg==", + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@prisma/studio-core/-/studio-core-0.27.3.tgz", + "integrity": "sha512-AADjNFPdsrglxHQVTmHFqv6DuKQZ5WY4p5/gVFY017twvNrSwpLJ9lqUbYYxEu2W7nbvVxTZA8deJ8LseNALsw==", "license": "Apache-2.0", + "dependencies": { + "@radix-ui/react-toggle": "1.1.10", + "chart.js": "4.5.1" + }, "engines": { - "node": "^20.19 || ^22.12 || ^24.0", + "node": "^20.19 || ^22.12 || >=24.0", "pnpm": "8" }, "peerDependencies": { @@ -2949,6 +2882,145 @@ "react-dom": "^18.0.0 || ^19.0.0" } }, + "node_modules/@radix-ui/primitive": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.3.tgz", + "integrity": "sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg==", + "license": "MIT" + }, + "node_modules/@radix-ui/react-compose-refs": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.2.tgz", + "integrity": "sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-primitive": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.3.tgz", + "integrity": "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.2.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slot": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.2.3.tgz", + "integrity": "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle/-/react-toggle-1.1.10.tgz", + "integrity": "sha512-lS1odchhFTeZv3xwHH31YPObmJn8gOg7Lq12inrr0+BH/l3Tsq32VfjqH1oh80ARM3mlkfMic15n0kg4sD1poQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.3", + "@radix-ui/react-primitive": "2.1.3", + "@radix-ui/react-use-controllable-state": "1.2.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.2.tgz", + "integrity": "sha512-BjasUjixPFdS+NKkypcyyN5Pmg83Olst0+c6vGov0diwTEo6mgdqVR6hxcEgFuh4QrAs7Rc+9KuGJ9TVCj0Zzg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-effect-event": "0.0.2", + "@radix-ui/react-use-layout-effect": "1.1.1" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-effect-event": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.2.tgz", + "integrity": "sha512-Qp8WbZOBe+blgpuUT+lw2xheLP8q0oatc9UpmiemEICxGvFLYmHm9QowVZGHtJlGbS6A6yJ3iViad/2cVjnOiA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.1" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.1.tgz", + "integrity": "sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/@scarf/scarf": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz", @@ -2986,9 +3058,9 @@ } }, "node_modules/@sinclair/typebox": { - "version": "0.34.48", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.48.tgz", - "integrity": "sha512-kKJTNuK3AQOrgjjotVxMrCn1sUJwM76wMszfq1kdU4uYVJjvEWuFQ6HgvLt4Xz3fSmZlTOxJ/Ie13KnIcWQXFA==", + "version": "0.34.49", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.49.tgz", + "integrity": "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==", "dev": true, "license": "MIT" }, @@ -3003,9 +3075,9 @@ } }, "node_modules/@sinonjs/fake-timers": { - "version": "15.1.1", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.1.1.tgz", - "integrity": "sha512-cO5W33JgAPbOh07tvZjUOJ7oWhtaqGHiZw+11DPbyqh2kHTBc3eF/CjJDeQ4205RLQsX6rxCuYOroFQwl7JDRw==", + "version": "15.3.2", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-15.3.2.tgz", + "integrity": "sha512-mrn35Jl2pCpns+mE3HaZa1yPN5EYCRgiMI+135COjr2hr8Cls9DXqIZ57vZe2cz7y2XVSq92tcs6kGQcT1J8Rw==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -3263,19 +3335,18 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.19.15", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.15.tgz", - "integrity": "sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg==", + "version": "22.19.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", + "integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==", "license": "MIT", "dependencies": { "undici-types": "~6.21.0" } }, "node_modules/@types/pg": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.18.0.tgz", - "integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==", - "dev": true, + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", "license": "MIT", "dependencies": { "@types/node": "*", @@ -3407,20 +3478,20 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.57.0.tgz", - "integrity": "sha512-qeu4rTHR3/IaFORbD16gmjq9+rEs9fGKdX0kF6BKSfi+gCuG3RCKLlSBYzn/bGsY9Tj7KE/DAQStbp8AHJGHEQ==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.58.1.tgz", + "integrity": "sha512-eSkwoemjo76bdXl2MYqtxg51HNwUSkWfODUOQ3PaTLZGh9uIWWFZIjyjaJnex7wXDu+TRx+ATsnSxdN9YWfRTQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.57.0", - "@typescript-eslint/type-utils": "8.57.0", - "@typescript-eslint/utils": "8.57.0", - "@typescript-eslint/visitor-keys": "8.57.0", + "@typescript-eslint/scope-manager": "8.58.1", + "@typescript-eslint/type-utils": "8.58.1", + "@typescript-eslint/utils": "8.58.1", + "@typescript-eslint/visitor-keys": "8.58.1", "ignore": "^7.0.5", "natural-compare": "^1.4.0", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3430,9 +3501,9 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.57.0", + "@typescript-eslint/parser": "^8.58.1", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { @@ -3446,16 +3517,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.57.0.tgz", - "integrity": "sha512-XZzOmihLIr8AD1b9hL9ccNMzEMWt/dE2u7NyTY9jJG6YNiNthaD5XtUHVF2uCXZ15ng+z2hT3MVuxnUYhq6k1g==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.58.1.tgz", + "integrity": "sha512-gGkiNMPqerb2cJSVcruigx9eHBlLG14fSdPdqMoOcBfh+vvn4iCq2C8MzUB89PrxOXk0y3GZ1yIWb9aOzL93bw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.57.0", - "@typescript-eslint/types": "8.57.0", - "@typescript-eslint/typescript-estree": "8.57.0", - "@typescript-eslint/visitor-keys": "8.57.0", + "@typescript-eslint/scope-manager": "8.58.1", + "@typescript-eslint/types": "8.58.1", + "@typescript-eslint/typescript-estree": "8.58.1", + "@typescript-eslint/visitor-keys": "8.58.1", "debug": "^4.4.3" }, "engines": { @@ -3467,18 +3538,18 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.57.0.tgz", - "integrity": "sha512-pR+dK0BlxCLxtWfaKQWtYr7MhKmzqZxuii+ZjuFlZlIGRZm22HnXFqa2eY+90MUz8/i80YJmzFGDUsi8dMOV5w==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.58.1.tgz", + "integrity": "sha512-gfQ8fk6cxhtptek+/8ZIqw8YrRW5048Gug8Ts5IYcMLCw18iUgrZAEY/D7s4hkI0FxEfGakKuPK/XUMPzPxi5g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.57.0", - "@typescript-eslint/types": "^8.57.0", + "@typescript-eslint/tsconfig-utils": "^8.58.1", + "@typescript-eslint/types": "^8.58.1", "debug": "^4.4.3" }, "engines": { @@ -3489,18 +3560,18 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.57.0.tgz", - "integrity": "sha512-nvExQqAHF01lUM66MskSaZulpPL5pgy5hI5RfrxviLgzZVffB5yYzw27uK/ft8QnKXI2X0LBrHJFr1TaZtAibw==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.58.1.tgz", + "integrity": "sha512-TPYUEqJK6avLcEjumWsIuTpuYODTTDAtoMdt8ZZa93uWMTX13Nb8L5leSje1NluammvU+oI3QRr5lLXPgihX3w==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.0", - "@typescript-eslint/visitor-keys": "8.57.0" + "@typescript-eslint/types": "8.58.1", + "@typescript-eslint/visitor-keys": "8.58.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3511,9 +3582,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.57.0.tgz", - "integrity": "sha512-LtXRihc5ytjJIQEH+xqjB0+YgsV4/tW35XKX3GTZHpWtcC8SPkT/d4tqdf1cKtesryHm2bgp6l555NYcT2NLvA==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.58.1.tgz", + "integrity": "sha512-JAr2hOIct2Q+qk3G+8YFfqkqi7sC86uNryT+2i5HzMa2MPjw4qNFvtjnw1IiA1rP7QhNKVe21mSSLaSjwA1Olw==", "dev": true, "license": "MIT", "engines": { @@ -3524,21 +3595,21 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.57.0.tgz", - "integrity": "sha512-yjgh7gmDcJ1+TcEg8x3uWQmn8ifvSupnPfjP21twPKrDP/pTHlEQgmKcitzF/rzPSmv7QjJ90vRpN4U+zoUjwQ==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.58.1.tgz", + "integrity": "sha512-HUFxvTJVroT+0rXVJC7eD5zol6ID+Sn5npVPWoFuHGg9Ncq5Q4EYstqR+UOqaNRFXi5TYkpXXkLhoCHe3G0+7w==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.0", - "@typescript-eslint/typescript-estree": "8.57.0", - "@typescript-eslint/utils": "8.57.0", + "@typescript-eslint/types": "8.58.1", + "@typescript-eslint/typescript-estree": "8.58.1", + "@typescript-eslint/utils": "8.58.1", "debug": "^4.4.3", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3549,13 +3620,13 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.57.0.tgz", - "integrity": "sha512-dTLI8PEXhjUC7B9Kre+u0XznO696BhXcTlOn0/6kf1fHaQW8+VjJAVHJ3eTI14ZapTxdkOmc80HblPQLaEeJdg==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.58.1.tgz", + "integrity": "sha512-io/dV5Aw5ezwzfPBBWLoT+5QfVtP8O7q4Kftjn5azJ88bYyp/ZMCsyW1lpKK46EXJcaYMZ1JtYj+s/7TdzmQMw==", "dev": true, "license": "MIT", "engines": { @@ -3567,21 +3638,21 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.57.0.tgz", - "integrity": "sha512-m7faHcyVg0BT3VdYTlX8GdJEM7COexXxS6KqGopxdtkQRvBanK377QDHr4W/vIPAR+ah9+B/RclSW5ldVniO1Q==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.58.1.tgz", + "integrity": "sha512-w4w7WR7GHOjqqPnvAYbazq+Y5oS68b9CzasGtnd6jIeOIeKUzYzupGTB2T4LTPSv4d+WPeccbxuneTFHYgAAWg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.57.0", - "@typescript-eslint/tsconfig-utils": "8.57.0", - "@typescript-eslint/types": "8.57.0", - "@typescript-eslint/visitor-keys": "8.57.0", + "@typescript-eslint/project-service": "8.58.1", + "@typescript-eslint/tsconfig-utils": "8.58.1", + "@typescript-eslint/types": "8.58.1", + "@typescript-eslint/visitor-keys": "8.58.1", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3591,7 +3662,7 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { @@ -3605,9 +3676,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", - "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3618,13 +3689,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.2" + "brace-expansion": "^5.0.5" }, "engines": { "node": "18 || 20 || >=22" @@ -3634,16 +3705,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.57.0.tgz", - "integrity": "sha512-5iIHvpD3CZe06riAsbNxxreP+MuYgVUsV0n4bwLH//VJmgtt54sQeY2GszntJ4BjYCpMzrfVh2SBnUQTtys2lQ==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.58.1.tgz", + "integrity": "sha512-Ln8R0tmWC7pTtLOzgJzYTXSCjJ9rDNHAqTaVONF4FEi2qwce8mD9iSOxOpLFFvWp/wBFlew0mjM1L1ihYWfBdQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.57.0", - "@typescript-eslint/types": "8.57.0", - "@typescript-eslint/typescript-estree": "8.57.0" + "@typescript-eslint/scope-manager": "8.58.1", + "@typescript-eslint/types": "8.58.1", + "@typescript-eslint/typescript-estree": "8.58.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3654,17 +3725,17 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.57.0.tgz", - "integrity": "sha512-zm6xx8UT/Xy2oSr2ZXD0pZo7Jx2XsCoID2IUh9YSTFRu7z+WdwYTRk6LhUftm1crwqbuoF6I8zAFeCMw0YjwDg==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.58.1.tgz", + "integrity": "sha512-y+vH7QE8ycjoa0bWciFg7OpFcipUuem1ujhrdLtq1gByKwfbC7bPeKsiny9e0urg93DqwGcHey+bGRKCnF1nZQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.0", + "@typescript-eslint/types": "8.58.1", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -3801,6 +3872,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -3815,6 +3889,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -3829,6 +3906,9 @@ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -3843,6 +3923,9 @@ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -3857,6 +3940,9 @@ "riscv64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -3871,6 +3957,9 @@ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -3885,6 +3974,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -3899,6 +3991,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -4017,7 +4112,6 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", - "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -4084,9 +4178,9 @@ } }, "node_modules/anymatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -4393,9 +4487,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.10.7", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.7.tgz", - "integrity": "sha512-1ghYO3HnxGec0TCGBXiDLVns4eCSx4zJpxnHrlqFQajmhfKMQBzUGDdkMK7fUW7PTHTeLf+j87aTuKuuwWzMGw==", + "version": "2.10.18", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.18.tgz", + "integrity": "sha512-VSnGQAOLtP5mib/DPyg2/t+Tlv65NTBz83BJBJvmLVHHuKJVaDOBvJJykiT5TR++em5nfAySPccDZDa4oSrn8A==", "dev": true, "license": "Apache-2.0", "bin": { @@ -4405,6 +4499,12 @@ "node": ">=6.0.0" } }, + "node_modules/better-result": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/better-result/-/better-result-2.8.2.tgz", + "integrity": "sha512-YOf0VSj5nUPI27doTtXF+BBnsiRq3qY7avHqfIWnppxTLGyvkLq1QV2RTxkwoZwJ60ywLfZ0raFF4J/G886i7A==", + "license": "MIT" + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -4470,9 +4570,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -4493,9 +4593,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", - "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "version": "4.28.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", + "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", "dev": true, "funding": [ { @@ -4513,11 +4613,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.9.0", - "caniuse-lite": "^1.0.30001759", - "electron-to-chromium": "^1.5.263", - "node-releases": "^2.0.27", - "update-browserslist-db": "^1.2.0" + "baseline-browser-mapping": "^2.10.12", + "caniuse-lite": "^1.0.30001782", + "electron-to-chromium": "^1.5.328", + "node-releases": "^2.0.36", + "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" @@ -4588,13 +4688,13 @@ "license": "MIT" }, "node_modules/bullmq": { - "version": "5.71.0", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.71.0.tgz", - "integrity": "sha512-aeNWh4drsafSKnAJeiNH/nZP/5O8ZdtdMbnOPZmpjXj7NZUP5YC901U3bIH41iZValm7d1i3c34ojv7q31m30w==", + "version": "5.73.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.73.4.tgz", + "integrity": "sha512-Q+NeFLtdKSD3GDPYSX4pH+Mc9E4OZVKimXwrnZ5WmndNy31COMy4vQV9zfhgfHGSUFrlpsBicfKYbSjx9FbO+A==", "license": "MIT", "dependencies": { "cron-parser": "4.9.0", - "ioredis": "5.9.3", + "ioredis": "5.10.1", "msgpackr": "1.11.5", "node-abort-controller": "3.1.1", "semver": "7.7.4", @@ -4733,9 +4833,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001778", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001778.tgz", - "integrity": "sha512-PN7uxFL+ExFJO61aVmP1aIEG4i9whQd4eoSCebav62UwDyp5OHh06zN4jqKSMePVgxHifCw1QJxdRkA1Pisekg==", + "version": "1.0.30001787", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001787.tgz", + "integrity": "sha512-mNcrMN9KeI68u7muanUpEejSLghOKlVhRqS/Za2IeyGllJ9I9otGpR9g3nsw7n4W378TE/LyIteA0+/FOZm4Kg==", "dev": true, "funding": [ { @@ -4786,18 +4886,16 @@ "license": "MIT", "peer": true }, - "node_modules/chevrotain": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-10.5.0.tgz", - "integrity": "sha512-Pkv5rBY3+CsHOYfV5g/Vs5JY9WTHHDEKOlohI2XeygaZhUeqhAlldZ8Hz9cRmxu709bvS08YzxHdTPHhffc13A==", - "license": "Apache-2.0", + "node_modules/chart.js": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", + "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", + "license": "MIT", "dependencies": { - "@chevrotain/cst-dts-gen": "10.5.0", - "@chevrotain/gast": "10.5.0", - "@chevrotain/types": "10.5.0", - "@chevrotain/utils": "10.5.0", - "lodash": "4.17.21", - "regexp-to-ast": "0.5.0" + "@kurkle/color": "^0.3.0" + }, + "engines": { + "pnpm": ">=8" } }, "node_modules/chokidar": { @@ -5311,9 +5409,9 @@ } }, "node_modules/conventional-changelog-angular": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/conventional-changelog-angular/-/conventional-changelog-angular-8.3.0.tgz", - "integrity": "sha512-DOuBwYSqWzfwuRByY9O4oOIvDlkUCTDzfbOgcSbkY+imXXj+4tmrEFao3K+FxemClYfYnZzsvudbwrhje9VHDA==", + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/conventional-changelog-angular/-/conventional-changelog-angular-8.3.1.tgz", + "integrity": "sha512-6gfI3otXK5Ph5DfCOI1dblr+kN3FAm5a97hYoQkqNZxOaYa5WKfXH+AnpsmS+iUH2mgVC2Cg2Qw9m5OKcmNrIg==", "dev": true, "license": "ISC", "dependencies": { @@ -5324,9 +5422,9 @@ } }, "node_modules/conventional-changelog-conventionalcommits": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/conventional-changelog-conventionalcommits/-/conventional-changelog-conventionalcommits-9.3.0.tgz", - "integrity": "sha512-kYFx6gAyjSIMwNtASkI3ZE99U1fuVDJr0yTYgVy+I2QG46zNZfl2her+0+eoviG82c5WQvW1jMt1eOQTeJLodA==", + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/conventional-changelog-conventionalcommits/-/conventional-changelog-conventionalcommits-9.3.1.tgz", + "integrity": "sha512-dTYtpIacRpcZgrvBYvBfArMmK2xvIpv2TaxM0/ZI5CBtNUzvF2x0t15HsbRABWprS6UPmvj+PzHVjSx4qAVKyw==", "dev": true, "license": "ISC", "dependencies": { @@ -5344,9 +5442,9 @@ "license": "ISC" }, "node_modules/conventional-commits-parser": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-6.3.0.tgz", - "integrity": "sha512-RfOq/Cqy9xV9bOA8N+ZH6DlrDR+5S3Mi0B5kACEjESpE+AviIpAptx9a9cFpWCCvgRtWT+0BbUw+e1BZfts9jg==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-6.4.0.tgz", + "integrity": "sha512-tvRg7FIBNlyPzjdG8wWRlPHQJJHI7DylhtRGeU9Lq+JuoPh5BKpPRX83ZdLrvXuOSu5Eo/e7SzOQhU4Hd2Miuw==", "dev": true, "license": "MIT", "dependencies": { @@ -5434,13 +5532,13 @@ } }, "node_modules/cosmiconfig-typescript-loader": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-6.2.0.tgz", - "integrity": "sha512-GEN39v7TgdxgIoNcdkRE3uiAzQt3UXLyHbRHD6YoL048XAeOomyxaP+Hh/+2C6C2wYjxJ2onhJcsQp+L4YEkVQ==", + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-6.3.0.tgz", + "integrity": "sha512-Akr82WH1Wfqatyiqpj8HDkO2o2KmJRu1FhKfSNJP3K4IdXwHfEyL7MOb62i1AGQVLtIQM+iCE9CGOtrfhR+mmA==", "dev": true, "license": "MIT", "dependencies": { - "jiti": "^2.6.1" + "jiti": "2.6.1" }, "engines": { "node": ">=v18" @@ -5602,9 +5700,9 @@ } }, "node_modules/defu": { - "version": "6.1.4", - "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.4.tgz", - "integrity": "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==", + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.7.tgz", + "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", "license": "MIT" }, "node_modules/delayed-stream": { @@ -5799,9 +5897,9 @@ "license": "MIT" }, "node_modules/effect": { - "version": "3.18.4", - "resolved": "https://registry.npmjs.org/effect/-/effect-3.18.4.tgz", - "integrity": "sha512-b1LXQJLe9D11wfnOKAk3PKxuqYshQ0Heez+y5pnkd3jLj1yx9QhM72zZ9uUrOQyNvrs2GZZd/3maL0ZV18YuDA==", + "version": "3.20.0", + "resolved": "https://registry.npmjs.org/effect/-/effect-3.20.0.tgz", + "integrity": "sha512-qMLfDJscrNG8p/aw+IkT9W7fgj50Z4wG5bLBy0Txsxz8iUHjDIkOgO3SV0WZfnQbNG2VJYb0b+rDLMrhM4+Krw==", "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.0.0", @@ -5809,9 +5907,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.313", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.313.tgz", - "integrity": "sha512-QBMrTWEf00GXZmJyx2lbYD45jpI3TUFnNIzJ5BBc8piGUDwMPa1GV6HJWTZVvY/eiN3fSopl7NRbgGp9sZ9LTA==", + "version": "1.5.335", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.335.tgz", + "integrity": "sha512-q9n5T4BR4Xwa2cwbrwcsDJtHD/enpQ5S1xF1IAtdqf5AAgqDFmR/aakqH3ChFdqd/QXJhS3rnnXFtexU7rax6Q==", "dev": true, "license": "ISC" }, @@ -6436,16 +6534,15 @@ "license": "MIT" }, "node_modules/fast-copy": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.0.2.tgz", - "integrity": "sha512-ybA6PDXIXOXivLJK/z9e+Otk7ve13I4ckBvGO5I2RRmBU1gMHLVDJYEuJYhGwez7YNlYji2M2DvVU+a9mSFDlw==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.0.3.tgz", + "integrity": "sha512-58apWr0GUiDFM8+3afrO6eYwJBn9ZAhDOzG3L+/9llab/haCARS2UIfffmOurYLwbgDRs8n0rfr6qAAPEAuAQw==", "license": "MIT" }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, "license": "MIT" }, "node_modules/fast-glob": { @@ -6508,7 +6605,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", - "dev": true, "funding": [ { "type": "github", @@ -6700,9 +6796,9 @@ } }, "node_modules/flatted": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.1.tgz", - "integrity": "sha512-IxfVbRFVlV8V/yRaGzk0UVIcsKKHMSfYw66T/u4nTwlWteQePsxe//LjudR1AMX4tZW3WFCh3Zqa/sjlqpbURQ==", + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", "dev": true, "license": "ISC" }, @@ -6928,9 +7024,9 @@ } }, "node_modules/get-tsconfig": { - "version": "4.13.6", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.6.tgz", - "integrity": "sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw==", + "version": "4.13.7", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.7.tgz", + "integrity": "sha512-7tN6rFgBlMgpBML5j8typ92BKFi2sFQvIdpAqLA2beia5avZDrMs0FLZiM5etShWq5irVyGcGMEA1jcDaK7A/Q==", "dev": true, "license": "MIT", "dependencies": { @@ -7142,9 +7238,9 @@ "license": "MIT" }, "node_modules/handlebars": { - "version": "4.7.8", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", - "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "version": "4.7.9", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", + "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", "dev": true, "license": "MIT", "dependencies": { @@ -7242,9 +7338,9 @@ } }, "node_modules/hono": { - "version": "4.11.4", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.11.4.tgz", - "integrity": "sha512-U7tt8JsyrxSRKspfhtLET79pU8K+tInj5QZXs1jSugO1Vq5dFj3kmZsRldo29mTBfcjDRVRXrEZ6LS63Cog9ZA==", + "version": "4.12.12", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.12.tgz", + "integrity": "sha512-p1JfQMKaceuCbpJKAPKVqyqviZdS0eUxH9v82oWo1kb9xjQ5wA6iP3FNVAPDFlz5/p7d45lO+BpSk1tuSZMF4Q==", "license": "MIT", "engines": { "node": ">=16.9.0" @@ -7480,9 +7576,9 @@ } }, "node_modules/ioredis": { - "version": "5.10.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.0.tgz", - "integrity": "sha512-HVBe9OFuqs+Z6n64q09PQvP1/R4Bm+30PAyyD4wIEqssh3v9L21QjCVk4kRLucMBcDokJTcLjsGeVRlq/nH6DA==", + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.10.1.tgz", + "integrity": "sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==", "license": "MIT", "dependencies": { "@ioredis/commands": "1.5.1", @@ -8125,9 +8221,9 @@ } }, "node_modules/jest-config/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", "dev": true, "license": "MIT", "dependencies": { @@ -8991,9 +9087,9 @@ } }, "node_modules/jest-runtime/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", "dev": true, "license": "MIT", "dependencies": { @@ -9615,7 +9711,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { @@ -9728,15 +9823,6 @@ "node": ">= 0.8.0" } }, - "node_modules/lilconfig": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", - "integrity": "sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==", - "license": "MIT", - "engines": { - "node": ">=10" - } - }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -9764,6 +9850,7 @@ "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true, "license": "MIT" }, "node_modules/lodash.camelcase": { @@ -10160,9 +10247,9 @@ } }, "node_modules/micromatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -10402,9 +10489,9 @@ "license": "MIT" }, "node_modules/node-addon-api": { - "version": "8.6.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.6.0.tgz", - "integrity": "sha512-gBVjCaqDlRUk0EwoPNKzIr9KkS9041G/q31IBShPs1Xz6UTA+EXdZADbzqAJQrpDRq71CIMnOP5VMut3SL0z5Q==", + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", + "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", "license": "MIT", "engines": { "node": "^18 || ^20 || >= 21" @@ -10450,9 +10537,9 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.36", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz", - "integrity": "sha512-TdC8FSgHz8Mwtw9g5L4gR/Sh9XhSP/0DEkQxfEFXOpiul5IiHgHan2VhYYb6agDSfp4KuvltmGApc8HMgUrIkA==", + "version": "2.0.37", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.37.tgz", + "integrity": "sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg==", "dev": true, "license": "MIT" }, @@ -10496,9 +10583,9 @@ } }, "node_modules/nodemon/node_modules/brace-expansion": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", - "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "dev": true, "license": "MIT", "dependencies": { @@ -10509,13 +10596,13 @@ } }, "node_modules/nodemon/node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.2" + "brace-expansion": "^5.0.5" }, "engines": { "node": "18 || 20 || >=22" @@ -10565,9 +10652,9 @@ } }, "node_modules/nypm/node_modules/citty": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/citty/-/citty-0.2.1.tgz", - "integrity": "sha512-kEV95lFBhQgtogAPlQfJJ0WGVSokvLr/UEoFPiKKOXF7pl98HfUVUD0ejsuTCld/9xH9vogSywZ5KqHzXrZpqg==", + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/citty/-/citty-0.2.2.tgz", + "integrity": "sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w==", "license": "MIT" }, "node_modules/object-assign": { @@ -10591,12 +10678,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/obuf": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", - "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", - "license": "MIT" - }, "node_modules/ohash": { "version": "2.0.11", "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.11.tgz", @@ -10946,9 +11027,9 @@ "license": "ISC" }, "node_modules/path-to-regexp": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", - "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", "license": "MIT" }, "node_modules/path-type": { @@ -11022,15 +11103,6 @@ "node": ">=4.0.0" } }, - "node_modules/pg-numeric": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pg-numeric/-/pg-numeric-1.0.2.tgz", - "integrity": "sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==", - "license": "ISC", - "engines": { - "node": ">=4" - } - }, "node_modules/pg-pool": { "version": "3.13.0", "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.13.0.tgz", @@ -11088,9 +11160,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -11289,9 +11361,9 @@ } }, "node_modules/postal-mime": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/postal-mime/-/postal-mime-2.7.3.tgz", - "integrity": "sha512-MjhXadAJaWgYzevi46+3kLak8y6gbg0ku14O1gO/LNOuay8dO+1PtcSGvAdgDR0DoIsSaiIA8y/Ddw6MnrO0Tw==", + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/postal-mime/-/postal-mime-2.7.4.tgz", + "integrity": "sha512-0WdnFQYUrPGGTFu1uOqD2s7omwua8xaeYGdO6rb88oD5yJ/4pPHDA4sdWqfD8wQVfCny563n/HQS7zTFft+f/g==", "license": "MIT-0" }, "node_modules/postgres": { @@ -11346,12 +11418,6 @@ "node": ">=0.10.0" } }, - "node_modules/postgres-range": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/postgres-range/-/postgres-range-1.1.4.tgz", - "integrity": "sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==", - "license": "MIT" - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -11391,16 +11457,16 @@ } }, "node_modules/prisma": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/prisma/-/prisma-7.5.0.tgz", - "integrity": "sha512-n30qZpWehaYQzigLjmuPisyEsvOzHt7bZeRyg8gZ5DvJo9FGjD+gNaY59Ns3hlLD5/jZH5GBeftIss0jDbUoLg==", + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-7.7.0.tgz", + "integrity": "sha512-HlgwRBt1uEFB9LStHL4HLYDvoi4BNu1rYA0hPG0zCAEyK9SaZBqp7E5Rjpc3Qh8Lex/ye/svoHZ0OWoFNhWxuQ==", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { - "@prisma/config": "7.5.0", - "@prisma/dev": "0.20.0", - "@prisma/engines": "7.5.0", - "@prisma/studio-core": "0.21.1", + "@prisma/config": "7.7.0", + "@prisma/dev": "0.24.3", + "@prisma/engines": "7.7.0", + "@prisma/studio-core": "0.27.3", "mysql2": "3.15.3", "postgres": "3.4.7" }, @@ -11594,9 +11660,9 @@ } }, "node_modules/react": { - "version": "19.2.4", - "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", - "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", + "version": "19.2.5", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.5.tgz", + "integrity": "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA==", "license": "MIT", "peer": true, "engines": { @@ -11604,16 +11670,16 @@ } }, "node_modules/react-dom": { - "version": "19.2.4", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", - "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", + "version": "19.2.5", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.5.tgz", + "integrity": "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag==", "license": "MIT", "peer": true, "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { - "react": "^19.2.4" + "react": "^19.2.5" } }, "node_modules/react-is": { @@ -11652,9 +11718,9 @@ } }, "node_modules/readdirp/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -11694,12 +11760,6 @@ "node": ">=4" } }, - "node_modules/regexp-to-ast": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/regexp-to-ast/-/regexp-to-ast-0.5.0.tgz", - "integrity": "sha512-tlbJqcMHnPKI9zSrystikWKwHkBqu2a/Sgw01h3zFjvYrMxEDYHzzoMZnUrbIfpTFEsoRnnviOXNCzFiSc54Qw==", - "license": "MIT" - }, "node_modules/remeda": { "version": "2.33.4", "resolved": "https://registry.npmjs.org/remeda/-/remeda-2.33.4.tgz", @@ -11723,20 +11783,19 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/resend": { - "version": "6.9.3", - "resolved": "https://registry.npmjs.org/resend/-/resend-6.9.3.tgz", - "integrity": "sha512-GRXjH9XZBJA+daH7bBVDuTShr22iWCxXA8P7t495G4dM/RC+d+3gHBK/6bz9K6Vpcq11zRQKmD+B+jECwQlyGQ==", + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/resend/-/resend-6.10.0.tgz", + "integrity": "sha512-i7CwZpYj4Oho1RxsTpLcCUkO08+HiL4NXrm6jLJ2WzJ89UGI8eROSieLONJA3hnUrf1OYnCyfq5F6POnHUMv1Q==", "license": "MIT", "dependencies": { - "postal-mime": "2.7.3", - "svix": "1.84.1" + "postal-mime": "2.7.4", + "svix": "1.88.0" }, "engines": { "node": ">=20" @@ -12052,13 +12111,13 @@ } }, "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" + "object-inspect": "^1.13.4" }, "engines": { "node": ">= 0.4" @@ -12430,9 +12489,9 @@ } }, "node_modules/svix": { - "version": "1.84.1", - "resolved": "https://registry.npmjs.org/svix/-/svix-1.84.1.tgz", - "integrity": "sha512-K8DPPSZaW/XqXiz1kEyzSHYgmGLnhB43nQCMeKjWGCUpLIpAMMM8kx3rVVOSm6Bo6EHyK1RQLPT4R06skM/MlQ==", + "version": "1.88.0", + "resolved": "https://registry.npmjs.org/svix/-/svix-1.88.0.tgz", + "integrity": "sha512-vm/JrrUd3bVyBE+3L33TIyVSs8gS5fYx7lrISvKlDJXTYX1ACH4REX8P1tHxsSKoZi/rvifM1t0XRc5Vc45THw==", "license": "MIT", "dependencies": { "standardwebhooks": "1.0.0", @@ -12515,9 +12574,9 @@ } }, "node_modules/swagger-ui-dist": { - "version": "5.32.0", - "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.32.0.tgz", - "integrity": "sha512-nKZB0OuDvacB0s/lC2gbge+RigYvGRGpLLMWMFxaTUwfM+CfndVk9Th2IaTinqXiz6Mn26GK2zriCpv6/+5m3Q==", + "version": "5.32.2", + "resolved": "https://registry.npmjs.org/swagger-ui-dist/-/swagger-ui-dist-5.32.2.tgz", + "integrity": "sha512-t6Ns52nS8LU2hqi0+rezMjFO1ZrCsCrnommXrU7Nfrg2va2dWahdvM6TuSwzdHpG29v6BHJyU1c/UWFhgVZzVQ==", "license": "Apache-2.0", "dependencies": { "@scarf/scarf": "=1.4.0" @@ -12589,23 +12648,23 @@ "license": "MIT" }, "node_modules/tinyexec": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.4.tgz", - "integrity": "sha512-u9r3uZC0bdpGOXtlxUIdwf9pkmvhqJdrVCH9fapQtgy/OeTTMZ1nqH7agtvEfmGui6e1XxjcdrlxvxJvc3sMqw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.1.tgz", + "integrity": "sha512-VKS/ZaQhhkKFMANmAOhhXVoIfBXblQxGX1myCQ2faQrfmobMftXeJPcZGp0gS07ocvGJWDLZGyOZDadDBqYIJg==", "license": "MIT", "engines": { "node": ">=18" } }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.16", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", + "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -12667,9 +12726,9 @@ } }, "node_modules/ts-api-utils": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.4.0.tgz", - "integrity": "sha512-3TaVTaAv2gTiMB35i3FiGJaRfwb3Pyn/j3m/bfAvGe8FB7CF6u+LMYqYlDh7reQf7UNvoTvdfAqHGmPGOSsPmA==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", + "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", "dev": true, "license": "MIT", "engines": { @@ -12680,19 +12739,19 @@ } }, "node_modules/ts-jest": { - "version": "29.4.6", - "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.6.tgz", - "integrity": "sha512-fSpWtOO/1AjSNQguk43hb/JCo16oJDnMJf3CdEGNkqsEX3t0KX96xvyX1D7PfLCpVoKu4MfVrqUkFyblYoY4lA==", + "version": "29.4.9", + "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.9.tgz", + "integrity": "sha512-LTb9496gYPMCqjeDLdPrKuXtncudeV1yRZnF4Wo5l3SFi0RYEnYRNgMrFIdg+FHvfzjCyQk1cLncWVqiSX+EvQ==", "dev": true, "license": "MIT", "dependencies": { "bs-logger": "^0.2.6", "fast-json-stable-stringify": "^2.1.0", - "handlebars": "^4.7.8", + "handlebars": "^4.7.9", "json5": "^2.2.3", "lodash.memoize": "^4.1.2", "make-error": "^1.3.6", - "semver": "^7.7.3", + "semver": "^7.7.4", "type-fest": "^4.41.0", "yargs-parser": "^21.1.1" }, @@ -12709,7 +12768,7 @@ "babel-jest": "^29.0.0 || ^30.0.0", "jest": "^29.0.0 || ^30.0.0", "jest-util": "^29.0.0 || ^30.0.0", - "typescript": ">=4.3 <6" + "typescript": ">=4.3 <7" }, "peerDependenciesMeta": { "@babel/core": { @@ -12916,16 +12975,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.57.0.tgz", - "integrity": "sha512-W8GcigEMEeB07xEZol8oJ26rigm3+bfPHxHvwbYUlu1fUDsGuQ7Hiskx5xGW/xM4USc9Ephe3jtv7ZYPQntHeA==", + "version": "8.58.1", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.58.1.tgz", + "integrity": "sha512-gf6/oHChByg9HJvhMO1iBexJh12AqqTfnuxscMDOVqfJW3htsdRJI/GfPpHTTcyeB8cSTUY2JcZmVgoyPqcrDg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.57.0", - "@typescript-eslint/parser": "8.57.0", - "@typescript-eslint/typescript-estree": "8.57.0", - "@typescript-eslint/utils": "8.57.0" + "@typescript-eslint/eslint-plugin": "8.58.1", + "@typescript-eslint/parser": "8.58.1", + "@typescript-eslint/typescript-estree": "8.58.1", + "@typescript-eslint/utils": "8.58.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -12936,7 +12995,7 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/uglify-js": { @@ -13127,9 +13186,9 @@ } }, "node_modules/validator": { - "version": "13.15.26", - "resolved": "https://registry.npmjs.org/validator/-/validator-13.15.26.tgz", - "integrity": "sha512-spH26xU080ydGggxRyR1Yhcbgx+j3y5jbNXk/8L+iRvdIEQ4uTRH2Sgf2dokud6Q4oAtsbNvJ1Ft+9xmm6IZcA==", + "version": "13.15.35", + "resolved": "https://registry.npmjs.org/validator/-/validator-13.15.35.tgz", + "integrity": "sha512-TQ5pAGhd5whStmqWvYF4OjQROlmv9SMFVt37qoCBdqRffuuklWYQlCNnEs2ZaIBD1kZRNnikiZOS1eqgkar0iw==", "license": "MIT", "engines": { "node": ">= 0.10" @@ -13366,9 +13425,9 @@ "license": "ISC" }, "node_modules/yaml": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", "license": "ISC", "bin": { "yaml": "bin.mjs" diff --git a/src/types/crawler.ts b/src/types/crawler.ts index fcce1e8..eeb1b91 100644 --- a/src/types/crawler.ts +++ b/src/types/crawler.ts @@ -28,4 +28,4 @@ export type CrawlerJobPayload = { session_id: string; settings: CrawlerRunSettings; input_defaults?: InputDefaultsConfig; -}; \ No newline at end of file +}; diff --git a/src/workers/crawler.worker.ts b/src/workers/crawler.worker.ts index 2a79882..46693a0 100644 --- a/src/workers/crawler.worker.ts +++ b/src/workers/crawler.worker.ts @@ -1,3 +1,7 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + import { workerRedis } from "@lib/redis"; import { cacheDel, cacheKeys, cacheSetString } from "@lib/cache"; import { logger } from "@services/logger.service"; From a5cd5e09a8175ffce0074e611b90bf4b5afd97bb Mon Sep 17 00:00:00 2001 From: Youssef Date: Wed, 15 Apr 2026 20:03:25 +0200 Subject: [PATCH 4/6] feat: add mapper file for crawler and use it --- jest.config.js | 1 + package-lock.json | 6 +- src/mappers/crawlSession.mapper.ts | 34 ++++++++++++ src/models/crawlSession.ts | 8 +-- src/queues/crawl.queue.ts | 2 +- src/services/crawlSession.service.ts | 82 ++++++++++++---------------- src/workers/crawler.worker.ts | 2 +- src/workers/fake_crawler.py | 2 + tsconfig.json | 1 + 9 files changed, 80 insertions(+), 58 deletions(-) create mode 100644 src/mappers/crawlSession.mapper.ts diff --git a/jest.config.js b/jest.config.js index edb0be9..dfea438 100644 --- a/jest.config.js +++ b/jest.config.js @@ -16,6 +16,7 @@ module.exports = { '^@lib/(.*)$': '/src/lib/$1', '^@services/(.*)$': '/src/services/$1', '^@utils/(.*)$': '/src/utils/$1', + '^@mappers/(.*)$': '/src/mappers/$1', '^@models/(.*)$': '/src/models/$1', '^@constants/(.*)$': '/src/constants/$1', '^@generated/(.*)$': '/src/generated/$1', diff --git a/package-lock.json b/package-lock.json index 958f9a2..c99662c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -996,9 +996,9 @@ } }, "node_modules/@electric-sql/pglite": { - "version": "0.3.15", - "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.3.15.tgz", - "integrity": "sha512-Cj++n1Mekf9ETfdc16TlDi+cDDQF0W7EcbyRHYOAeZdsAe8M/FJg18itDTSwyHfar2WIezawM9o0EKaRGVKygQ==", + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.4.1.tgz", + "integrity": "sha512-mZ9NzzUSYPOCnxHH1oAHPRzoMFJHY472raDKwXl/+6oPbpdJ7g8LsCN4FSaIIfkiCKHhb3iF/Zqo3NYxaIhU7Q==", "license": "Apache-2.0" }, "node_modules/@electric-sql/pglite-socket": { diff --git a/src/mappers/crawlSession.mapper.ts b/src/mappers/crawlSession.mapper.ts new file mode 100644 index 0000000..5d88d9c --- /dev/null +++ b/src/mappers/crawlSession.mapper.ts @@ -0,0 +1,34 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + +import { CrawlStatus as PrismaCrawlStatus, CrawlTriggerType as PrismaCrawlTriggerType } from "@generated/prisma/client"; +import { CrawlStatus, CrawlTriggerType } from "@models/crawlSession"; + +export const toDbCrawlStatus = (status: CrawlStatus): PrismaCrawlStatus => { + const key = CrawlStatus[status] as unknown as keyof typeof PrismaCrawlStatus; + return PrismaCrawlStatus[key] ?? PrismaCrawlStatus.UNSPECIFIED; +}; + +export const toDbCrawlTriggerType = (triggerType: CrawlTriggerType): PrismaCrawlTriggerType => { + const key = CrawlTriggerType[triggerType] as unknown as keyof typeof PrismaCrawlTriggerType; + return PrismaCrawlTriggerType[key] ?? PrismaCrawlTriggerType.UNSPECIFIED; +}; + +export const toDbCrawlStatusFilter = (status?: CrawlStatus): TDbStatus | undefined => { + if (status === undefined || status === CrawlStatus.UNSPECIFIED) return undefined; + return toDbCrawlStatus(status) as unknown as TDbStatus; +}; + +export const toDbCrawlTriggerTypeFilter = (triggerType?: CrawlTriggerType): TDbTriggerType | undefined => { + if (triggerType === undefined || triggerType === CrawlTriggerType.UNSPECIFIED) return undefined; + return toDbCrawlTriggerType(triggerType) as unknown as TDbTriggerType; +}; + +export const fromDbCrawlStatus = (status: TDbStatus): CrawlStatus => { + return (CrawlStatus[status as unknown as keyof typeof CrawlStatus] ?? CrawlStatus.UNSPECIFIED) as CrawlStatus; +}; + +export const fromDbCrawlTriggerType = (triggerType: TDbTriggerType): CrawlTriggerType => { + return (CrawlTriggerType[triggerType as unknown as keyof typeof CrawlTriggerType] ?? CrawlTriggerType.UNSPECIFIED) as CrawlTriggerType; +}; diff --git a/src/models/crawlSession.ts b/src/models/crawlSession.ts index 066680a..dc22467 100644 --- a/src/models/crawlSession.ts +++ b/src/models/crawlSession.ts @@ -32,8 +32,6 @@ export type GetSessionsQuery = ZodInfer; export { CrawlTriggerType, CrawlStatus }; - - export const CrawlConfigSchema = z.object({ maxStates: z.number().int().min(1).max(100000), maxDepth: z.number().int().min(1).max(1000), @@ -72,7 +70,7 @@ export const CrawlConfigSchema = z.object({ }).loose() satisfies ZodType; export const CreateCrawlSessionRequestSchema = z.object({ - triggerType: z.nativeEnum(CrawlTriggerType), + triggerType: z.enum(CrawlTriggerType), crawlConfig: CrawlConfigSchema, }) satisfies ZodType; @@ -87,8 +85,8 @@ export const CrawlSessionParamsSchema = z.object({ export const GetSessionsQuerySchema = z.object({ page: z.coerce.number().int().min(1).default(1), pageSize: z.coerce.number().int().min(1).max(100).default(25), - status: z.nativeEnum(CrawlStatus).optional(), - triggerType: z.nativeEnum(CrawlTriggerType).optional(), + status: z.enum(CrawlStatus).optional(), + triggerType: z.enum(CrawlTriggerType).optional(), }); diff --git a/src/queues/crawl.queue.ts b/src/queues/crawl.queue.ts index d459dd7..0e42ff2 100644 --- a/src/queues/crawl.queue.ts +++ b/src/queues/crawl.queue.ts @@ -5,7 +5,7 @@ import { Queue } from 'bullmq'; import redis from '@lib/redis'; -export const crawlQueue = new Queue('crawl-tasks', { +export const crawlQueue = new Queue('crawler', { connection: redis }); diff --git a/src/services/crawlSession.service.ts b/src/services/crawlSession.service.ts index 7da6b3d..e8a3c02 100644 --- a/src/services/crawlSession.service.ts +++ b/src/services/crawlSession.service.ts @@ -3,10 +3,17 @@ // See LICENSE file in the project root for full license information. import prisma from "@lib/prisma"; +import { CrawlStatus as PrismaCrawlStatus } from "@generated/prisma/client"; +import { + fromDbCrawlStatus, + fromDbCrawlTriggerType, + toDbCrawlStatusFilter, + toDbCrawlTriggerType, + toDbCrawlTriggerTypeFilter, +} from "@mappers/crawlSession.mapper"; import { type CrawlConfig, CrawlConfigSchema, - CrawlStatus, CrawlTriggerType, type ApplicationVersionCrawlSessionsResponse, type CrawlSessionData, @@ -17,36 +24,15 @@ import { CrawlerJobPayload } from "types/crawler"; import { toIso } from "@utils/date"; - - type DbCrawlSession = Awaited>; - type DbCrawlStatus = DbCrawlSession["status"]; type DbCrawlTriggerType = DbCrawlSession["triggerType"]; -const toDbStatusFilter = (status?: CrawlStatus): DbCrawlStatus | undefined => { - if (status === undefined || status === CrawlStatus.UNSPECIFIED) return undefined; - return CrawlStatus[status] as unknown as DbCrawlStatus; -}; - -const toDbTriggerTypeFilter = (triggerType?: CrawlTriggerType): DbCrawlTriggerType | undefined => { - if (triggerType === undefined || triggerType === CrawlTriggerType.UNSPECIFIED) return undefined; - return CrawlTriggerType[triggerType] as unknown as DbCrawlTriggerType; -}; - -const fromDbStatus = (status: DbCrawlStatus): CrawlStatus => { - return (CrawlStatus[status as unknown as keyof typeof CrawlStatus] ?? CrawlStatus.UNSPECIFIED) as CrawlStatus; -}; - -const fromDbTriggerType = (triggerType: DbCrawlTriggerType): CrawlTriggerType => { - return (CrawlTriggerType[triggerType as unknown as keyof typeof CrawlTriggerType] ?? CrawlTriggerType.UNSPECIFIED) as CrawlTriggerType; -}; - const mapSession = (session: DbCrawlSession): CrawlSessionData => ({ id: session.id, appVersionId: session.appVersionId, - status: fromDbStatus(session.status), - triggerType: fromDbTriggerType(session.triggerType), + status: fromDbCrawlStatus(session.status), + triggerType: fromDbCrawlTriggerType(session.triggerType), crawlConfig: (() => { const parsed = CrawlConfigSchema.safeParse(session.config); return parsed.success ? parsed.data : undefined; @@ -64,8 +50,8 @@ export async function getSessions( query: GetSessionsQuery ): Promise { const { page, pageSize, status, triggerType } = query; - const dbStatus = toDbStatusFilter(status); - const dbTriggerType = toDbTriggerTypeFilter(triggerType); + const dbStatus = toDbCrawlStatusFilter(status); + const dbTriggerType = toDbCrawlTriggerTypeFilter(triggerType); const [sessions, totalCount] = await Promise.all([ prisma.crawlSession.findMany({ where: { @@ -115,7 +101,7 @@ export async function createSession( const newSession = await prisma.crawlSession.create({ data: { appVersionId: versionId, - triggerType: CrawlTriggerType[triggerType] as unknown as DbCrawlTriggerType, + triggerType: toDbCrawlTriggerType(triggerType) as unknown as DbCrawlTriggerType, config: persistedConfig, } }); @@ -141,10 +127,10 @@ export async function startSession(sessionId: string): Promise { where: { id: sessionId } }); - if (session.status === CrawlStatus[CrawlStatus.NEW]) { + if (session.status === PrismaCrawlStatus.NEW) { await prisma.crawlSession.update({ where: { id: sessionId }, - data: { status: CrawlStatus[CrawlStatus.QUEUED] } + data: { status: PrismaCrawlStatus.QUEUED } }); try { @@ -152,7 +138,7 @@ export async function startSession(sessionId: string): Promise { } catch (error) { await prisma.crawlSession.update({ where: { id: sessionId }, - data: { status: CrawlStatus[CrawlStatus.NEW] } + data: { status: PrismaCrawlStatus.NEW } }); throw error; } @@ -160,10 +146,10 @@ export async function startSession(sessionId: string): Promise { return; } - if (session.status === CrawlStatus[CrawlStatus.PAUSED]) { + if (session.status === PrismaCrawlStatus.PAUSED) { await prisma.crawlSession.update({ where: { id: sessionId }, - data: { status: CrawlStatus[CrawlStatus.RUNNING] } + data: { status: PrismaCrawlStatus.RUNNING } }); return; } @@ -176,18 +162,18 @@ export async function abortSession(sessionId: string): Promise { where: { id: sessionId } }); - if (session.status !== CrawlStatus[CrawlStatus.RUNNING] && session.status !== CrawlStatus[CrawlStatus.PAUSED] - && session.status !== CrawlStatus[CrawlStatus.QUEUED]) { + if (session.status !== PrismaCrawlStatus.RUNNING && session.status !== PrismaCrawlStatus.PAUSED + && session.status !== PrismaCrawlStatus.QUEUED) { throw new Error(`Cannot abort session with status ${session.status}`); } - if (session.status === CrawlStatus[CrawlStatus.QUEUED]) { + if (session.status === PrismaCrawlStatus.QUEUED) { await removeCrawlJob(sessionId); } await prisma.crawlSession.update({ where: { id: sessionId }, - data: { status: CrawlStatus[CrawlStatus.ABORTED] } + data: { status: PrismaCrawlStatus.ABORTED } }); }; @@ -196,13 +182,13 @@ export async function pauseSession(sessionId: string): Promise { where: { id: sessionId } }); - if (session.status !== CrawlStatus[CrawlStatus.RUNNING]) { + if (session.status !== PrismaCrawlStatus.RUNNING) { throw new Error(`Cannot pause session with status ${session.status}`); } await prisma.crawlSession.update({ where: { id: sessionId }, - data: { status: CrawlStatus[CrawlStatus.PAUSED] } + data: { status: PrismaCrawlStatus.PAUSED } }); }; @@ -210,10 +196,10 @@ export async function markQueuedSessionRunning(sessionId: string): Promise const result = await prisma.crawlSession.updateMany({ where: { id: sessionId, - status: CrawlStatus[CrawlStatus.QUEUED], + status: PrismaCrawlStatus.QUEUED, }, data: { - status: CrawlStatus[CrawlStatus.RUNNING], + status: PrismaCrawlStatus.RUNNING, startedAt: new Date(), }, }); @@ -230,7 +216,7 @@ export async function markSessionCompleted(sessionId: string): Promise { await prisma.crawlSession.update({ where: { id: sessionId }, data: { - status: CrawlStatus[CrawlStatus.COMPLETED], + status: PrismaCrawlStatus.COMPLETED, finishedAt: new Date(), error: null, }, @@ -241,7 +227,7 @@ export async function markSessionFailed(sessionId: string, errorMessage: string) await prisma.crawlSession.update({ where: { id: sessionId }, data: { - status: CrawlStatus[CrawlStatus.FAILED], + status: PrismaCrawlStatus.FAILED, finishedAt: new Date(), error: errorMessage, }, @@ -253,17 +239,17 @@ export async function isSessionAborted(sessionId: string): Promise { where: { id: sessionId }, select: { status: true }, }); - return session?.status === CrawlStatus[CrawlStatus.ABORTED]; + return session?.status === PrismaCrawlStatus.ABORTED; } export async function markSessionCompletedIfRunning(sessionId: string): Promise { const result = await prisma.crawlSession.updateMany({ where: { id: sessionId, - status: CrawlStatus[CrawlStatus.RUNNING], + status: PrismaCrawlStatus.RUNNING, }, data: { - status: CrawlStatus[CrawlStatus.COMPLETED], + status: PrismaCrawlStatus.COMPLETED, finishedAt: new Date(), error: null, }, @@ -275,10 +261,10 @@ export async function markSessionFailedIfRunning(sessionId: string, errorMessage const result = await prisma.crawlSession.updateMany({ where: { id: sessionId, - status: CrawlStatus[CrawlStatus.RUNNING], + status: PrismaCrawlStatus.RUNNING, }, data: { - status: CrawlStatus[CrawlStatus.FAILED], + status: PrismaCrawlStatus.FAILED, finishedAt: new Date(), error: errorMessage, }, @@ -290,7 +276,7 @@ export async function markSessionFinishedAtIfAborted(sessionId: string): Promise await prisma.crawlSession.updateMany({ where: { id: sessionId, - status: CrawlStatus[CrawlStatus.ABORTED], + status: PrismaCrawlStatus.ABORTED, finishedAt: null, }, data: { diff --git a/src/workers/crawler.worker.ts b/src/workers/crawler.worker.ts index 46693a0..545a67b 100644 --- a/src/workers/crawler.worker.ts +++ b/src/workers/crawler.worker.ts @@ -189,7 +189,7 @@ async function runCrawler(sessionId: string): Promise { } new Worker( - "crawl-tasks", + "crawler", async (job) => { if (job.name !== "crawl") return; await runCrawler(job.data.sessionId); diff --git a/src/workers/fake_crawler.py b/src/workers/fake_crawler.py index c930a27..0686c3a 100644 --- a/src/workers/fake_crawler.py +++ b/src/workers/fake_crawler.py @@ -26,11 +26,13 @@ def command_listener(): threading.Thread(target=command_listener, daemon=True).start() def run_crawler(): + i = 0 while state["running"]: if state["paused"]: time.sleep(1) # Wait and check again continue print(f"Crawling... {i+1}/30") + i += 1 time.sleep(1) if __name__ == "__main__": diff --git a/tsconfig.json b/tsconfig.json index c87e434..d26d1b2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,6 +17,7 @@ "@lib/*": ["lib/*"], "@services/*": ["services/*"], "@utils/*": ["utils/*"], + "@mappers/*": ["mappers/*"], "@generated/*": ["generated/*"], "@models/*": ["models/*"], "@constants/*": ["constants/*"], From 5a5075438c35be89a3930feeb8bc2e5ac242a161 Mon Sep 17 00:00:00 2001 From: Youssef Date: Fri, 17 Apr 2026 22:49:36 +0200 Subject: [PATCH 5/6] refactor: remove the crawler worker and move it to crawler repository --- docker-compose.yml | 3 +- package.json | 1 - .../api_crawler_smoke_test.cpython-313.pyc | Bin 0 -> 20970 bytes .../api_crawler_smoke_test.cpython-314.pyc | Bin 0 -> 23473 bytes scripts/api_crawler_smoke_test.py | 411 ++++++++++++++++++ src/queues/crawl.queue.ts | 22 +- src/queues/stream/crawlStream.ts | 32 ++ src/services/crawlSession.service.ts | 145 +----- src/types/crawler.ts | 11 +- src/workers/crawler.worker.ts | 200 --------- src/workers/fake_crawler.py | 46 -- 11 files changed, 459 insertions(+), 412 deletions(-) create mode 100644 scripts/__pycache__/api_crawler_smoke_test.cpython-313.pyc create mode 100644 scripts/__pycache__/api_crawler_smoke_test.cpython-314.pyc create mode 100644 scripts/api_crawler_smoke_test.py create mode 100644 src/queues/stream/crawlStream.ts delete mode 100644 src/workers/crawler.worker.ts delete mode 100644 src/workers/fake_crawler.py diff --git a/docker-compose.yml b/docker-compose.yml index 168caac..29d371a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,10 +43,11 @@ services: condition: service_healthy redis: condition: service_healthy - db: image: postgres:18 container_name: coverit-postgres + ports: + - "5432:5432" environment: - POSTGRES_USER=${DB_USER:-postgres} - POSTGRES_PASSWORD=${DB_PASSWORD:-postgres} diff --git a/package.json b/package.json index 71d6e4a..048ac86 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,6 @@ "build": "prisma generate && tsc && tsc-alias", "start": "node dist/index.js", "worker:email": "ts-node -r tsconfig-paths/register src/workers/email.worker.ts", - "worker:crawler": "ts-node -r tsconfig-paths/register src/workers/crawler.worker.ts", "postinstall": "prisma generate", "db:migrate": "prisma migrate dev", "db:deploy": "prisma migrate deploy", diff --git a/scripts/__pycache__/api_crawler_smoke_test.cpython-313.pyc b/scripts/__pycache__/api_crawler_smoke_test.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d7c906321065baaa5add23543e2f9b9bb2e6dce4 GIT binary patch literal 20970 zcmeHvYgk)Jme|$%>VX78yk7!@L70~TyKVdwARYz-*44GK!PW@rf@KSdzE`-7x_g$M zO?Gg55}3_Qn%?P+>Fj)I=0`GY{)El$W|-;DhRy!itGV7mdxx*n-^SU*`|S^U+M7&z z@*}6}>PiCBZF~CLFZuGNY^v&3oqC-*bKhbmTuLp7_$p@!AsP?rix94d$O7Y(f8qLDRTG_j_O zIc$!dpj4A)N)3OSNpq)`(v}jp)vP6znhUA9kgCI}d8t$@q*@`B#Hsm|9+x(Bswo3b zu~7>6XQWImDk=x^%$Z|D zpzdzMs+MxJkZfrR8kMiZaAlB!FlL36BRHiTe<`VY{8ceoNi}qh z6NKMP5S7GaRShv&Q9~S0P3duk6k!xIifUrgt|R<NzSyGB6u#O;63VVwyOssv@Qp&ZZ}*%!pGhs$7u;QQaE~ z#6)s*E*6SL{9#e!ou3OcqB`QAWkg+&nfA|zW1^O2V)JYy0OXi~_1@9Ejd!*ifw5KNo7{0stA~S|RQWu>RM=4C~`& zqjwlzjNxJ}bASsURLSN+VH5mus0WM0E@@cSzpYKOIfQ>vaZFS!09GTSYJuTIb&Q>7ys4}})Oq)gvVPrlQd>Ylm-yZSEp?fN6ti+4;6} zrT0PWVoy?McG!~~*H>|k{R^6fCo zoU)2=5c2wDjgals97M1x6p4v?_q_mv7iu=avsF++CV2_RB*rlmVI7cHf%4tp9fso7 z+K0|}oez(`3#bHS?=HY&genqlt60x8x8`2>+hxfedZ3yko^`2$82Dvw z10s9dh(>SwoI`~Fje=rg)l{$g7m9k-lS)Vcrlg4hCy9vasc3LP)Wt%xOmrUj<}4Gt z6%C52c{a=*1xU6DdClOZ2^0|~nsiK{SHMRCKcWFKs%%q4w6tmpy7*k|sbn%cjHgoR zT&Pz`YbiAvI;~9eSRhMELo3Zh2g=o=+zBms*m=@KE2j{I2le=X8!s6Dng{WO15Rd1 zlT03YN8} z4aIQ@SncYdKjs&8w-|qrVL7%1dietdUe+SR)CL); z92fJ)<~hvEL=D5TQC8HW#@#rooSqX+p7{u3MpB9z7b9uUF#dL$aL?IR=+go2#CD=I zw5~)I^a?f1&0$F+DnKus^Dkg#int+2_Oz44kHL2<7MsI7mmLKNgdT1dR>vZdG+GvW zKOu8@t7A{4v}hOm_p}6QO6Wl7WSH-cKh)^slER?G>q>SLWLS=Y2q zulMt{D9w@}?T-z)%Psd?b`17}!M>KiS@h@Se_H-$mAt{u8^#xV_SAaKkYYC{fBEM9 zn>#tRiJaQC9v+3KHoZw};XBT6JO9bi9jhZ@b*x<#tj$Y3$^4>sPJjFKzpH&@Kehqh ziLFjPzyE(|2y@p?&MEl+tEXDRWL-XU|ICiDI$^B-j8JLD6v@V;KXm-ivE*N#zCZoo z)^hZIbam#D`N&51lIr88)<;%o_>r|?!@H#0HJ7f^YsWq~@&1XmhV|<8zV(wE$2O=9 z_h#PaE1OQftXD9*w>x=q^jEvaJgFgj!e~dL7zP8Sed|}&2R2+Afem`IdegPp!IzyE z%onzTJbCd~NKy?cKIeJ>?LTg-=&2`ul5@VNQ1zin5AhER4LuIshjtZ;JFs70+1H@@ zNaJd7H>>`Ab2WVYiMgP!R`n;f8i+qZ44FLvURq?qR1%2MW0CcbfgL_ir#eh1(P*a# zB911$Opm2hr&JNu^VA0q?4b_2@pej$g@&3|A11ESLgNsTMQJG5`JPfnG*A|4xCb1q z`3mA7U6%(b#0Q)QDX#|{Fh7w0pbBUY6JKd@0GbFu93q%sWTYSr$cQE*{3t!tb%?kE z#%{oj*FYmCZ=f|G5{+tL`_Hg?txF56K0TI0M|-&NL#$q#)fb7?%PndTVfDDJELP7R zgLdPp#@0rs60Fir)qVZ<&u_7#+Q5_zYhob7^`)6BvF z4Z)s6-g)F*K%NVE-N-`=jqODq(z4k;W;SN0Nk zoDRGcKi$Vok%rCEkcn#MN-#Iw-Ov2i?`;CM- zCYx4x5W76!GEfKxHs#6@QPJvD!ift$*$}m1>4ZbcUI761DtH_)Tp$sNI`qEV(NM&P z_7{_ojCNa`13WBhmrvh6y}}8`vef{{skPn@hTb3APzjFCgt3!XcS;-O?7pj^BxdSA zgUGMN)HA0%3K(%(^M&eCF3qk$Zv_ZaPS1ogL=YJ7M4FvZAShtrr?=D07t{|-NqKm^ zP+A2TR*l4S1|xU<;ZV>HnoyTLuIg%Sb!zb<7d3N!7ABnpgmR!G!WNDT++yG?OdDZh z;b;J_Y2YbHXTrcSZr^<9e4r=Jgd%JJ0N@mm;1H3(&Kop_;rqiors{;LdaYJ49a(fI zjk(Kbmd~utKYZii8)$*tSnpV`wJFBhfW%%A;X6n@F^?WR<0E?=FSe#M7k*27lM}|g5ua3mYRHQ*vI7NleX%Gz| zE);=1QzXD(nih+EJs#K9VISpnd1=bmea$yQU+Q*yL=_w_iz?u}_;?e`dr^zm8z+)X zB*?uMirvB_4`-feN#K)C>GTtNfAGw+$QOySv%qoUjCAzuOYPhi0XBx<{}{aVqA7l? z%Uh{?FMmf@k%gN0Ss!;9};T+dtYTtBiF-zjew%G;AgWxs9w zR6`UThwWHy-tyO126?jLF=<)RJTT+q*0n_&ZvO2b`QQ(~}=gNG8yN#*R))-G~xs1ZeQAvI~?rKZ#C) za17RGB~YgW=#o-WYOLz3C|!#NXist2nw49OsK$@h)isC`vk|Jp|ttkG{>6OdhI-Sh7Ee-6c2vZSITX9L! znx8BvOcoYF`lm)MJ`^w$M$>Zr{rU$^d=>DUUxyRu`)}}M*{-3GFFCu(Y+mJy+=8Kx zSNBPz2VF@!bS#0Hc%%@kF-QhrhjyGYTa9F~T}nl%)9G2v9uY@6U72D09w32(u7;p= zGBZSMEKQ&VzzkJiMp!ar@CXnonGpaU!@)KPB6E}IL8Cn2U@xXedojw;0lVgm(1IJ; zuUfAO@9{F&Q$H6QIcaQYvU&pd!C!HN2YaH;sy&y=OV5`sE8R-vr_;%awD`+{uu%mv zM7rPliL~I$f-Izpo`EzRT4ym;@(iRgBlf6Ls!Yz0n6lE#v(j@WfPN&0?x)l6=B)Ip ztaM9e-#n4tzaMtUjILy96|JUqw4OH5M#|ozLpR^2(keCxZj(%#{x^WlqMIb_GRLqP z(M{E+`<6RVEkjZcI+v=Wj+|1>0~?IwMXVE`97v9~PB`SaTsGYvS~XTry_}y3sd{;S z;GX?8>7ISUDd$s8+PZ(6@Z$KO8lbO@Ey`!Gkdy8eY}O}`f{PuDO}B4s`E(w2vmE zS~CSAnAXr`6dce31zYPy6PrO7r$);MeEnwK7DEnWKs;~b-FzQQCrk4 zm6~3PmLX(XJQVH|aT>NsU99G1a`7PbhHI=b$K zbTC+1mq0J3>&BFq;6xp5UpZx@E=>P?P`1`*4H0neGERiaOxrEeqxA8UP-W!Qa;? z-%FG`O#AUuQ*sS5uCwY7yj1;}eg!E;u9cbw>YHgHBS$7WJpPTKheE9?&4uz!WUaP4XBCDQm(GT%uzPRP?r(2ergsc?(> zpuVzC6Og#B#FM18I!ilY#YQ^xG9DvGv+x(CkJ2S=L>6yqdZ~UkQFAZNJ#dfU{s`6{ za*^f}E3--&Y6;lj4XYW^BLI5_#e`3 z`!$-G$}hloh6nwYOfxgpGv5S5N4aVOZWF`6M++v9h?VB9K*npNkUCBmqLRTDIPW}> zibGlI6}r&#CE6@5V*|L8&xiBF=J_S$Lm6F2oqASpsY0f>1+IdAa^zJe70h&K6r-ZeJp>vmCYAKWv?%aKtW+%&@_vOjz)%Eh`m+S=OCbA;Br$HW4; zjMq(hePb?)x;pCVja$87J~-y*xYwd=@K}3C^@Un7Zy)H0>ykU33oY3x^q9ep3V_L= zbsv!X65N!=4bWCIe)7=#1RWm7O&6GZ{#khM(GrNxX1YD*8NKN4@%m8JPe=e1%jDYR ztX?_~OeXg-FTr_Ts`B}I~gVz^x&CftR4=-R_8IIi&3lCHd z7atTf8rQV7w6z=)E9C64(cxj=;E31lx#EHgJ$KLONG~NS+QssH5Z=K{?oryC22h=7 zYZL2a5J35m+v6J>>=~lRd-N})V+zqBG zG#eK4=c3{8AQ-W-a9v4(ohV#ZidK~9m2A@Gv{Im`I}AG(3f!5W+n*)^@njp-g=|*hN4%I;?8xK>nPu1`(HNBV2X6VN!i>E^x-bP>4JWS1CXITG zWpIO(WC)9RN8Hrv2W#C{E;JLFpA*$%qm&n(ORPbbNY15J627U0hXPHI3f(eEz)~$> z@qvrRNo>d4*>+0Q;PnGG{(fZRuk(kI8SVlU%t;lz69PM3FwBK?uqKWhr7nb{U~3C2rN@~1&TcC;#w$`8d5kGQJjPU=e@!Kn5D&CcLmw5imAynww{V6L3ruh(U&J!ls*c zDGm&Vi8Td*Rm4JffvTH;_sxbPC>CU+b4|?bTx`J}@UyX|Ko|`AaTL~22|R9u8*Nx8 z@Zb%VWqha#L`1giM7R}4!k2TBqJGwY50MMJk|QD73vUT-iPlgg5S|YrS|9VrpmC_c z%G`SHcj98Xv zfaDm{6NOg{q9w-qBOEjyjYtV5Y`A*~y6c}|L_My73A$pUVH$eH-9m8?o4u$aJv>LG zBq{M<;G6wx%B}ORyp2-A3Lm@Zxp7U_&SgssIB7 zWZ#e4AQJ;o&CM%dSJi`FoTiZ;;_O^36b{=Z>o&W88tBG8hh?!nMFjT0TKtY(64Lf? z=nfNJz)gWf=jMUFXIkuTh_Q@4b}Q8|R3YU;1X^I{BN2eJH^{GGaqk$m%RaXN&sie& z=2<((23n*xxE6S-G7t7ixPFi_c7G%Yb^BxXP>i#C+`WSo(qnOQugmL_6*&7G7@)@N zu_*2c2NVFqWbcMRz}fvAP6YL!g^d8l<_EyrkBSHb%+?NS2E&5rlkm}+GWNqLz|(4n zS9^H)xBP%++~eRmWQ1{WhVbMdFJ2|J1DY);xJVO0Vl_ifXX7gS8GGDu7}NnmI~#8} z6bck7s1IK9WF+)Cbc8AO2@^~dKAlzgP4U4eo$C4dP!L}DfZr*qZZY@Zc`ZDuNg3?e zQ9bB(9E?m^WM2vU8G=PSAZ+K0%}ep)8a9rA)IoU0&B7gh%IKU%u{V*2-Y3E{X=KZa z4R_c@l%PgLCaQprATP`?bCNOJNy*9>=_*)?OKOuTCH{T*QFy-mm8D_8+QjOc55%+t zB_|*)09hvkZ<8QWqE*a6Iw`Uo#hR*U!k;P97tPXA@X6oM14+??2124G{Uy~Y8ekmQ zqz)Lf1;sNH$cY9?F~ug*h!wCS$Xh^IFi7ef&bovWRI}WSsK?3`?le3r1=UP6N!pkX zYj2`eQdN919?s}FzCn3*dULJgVkS}Z$Oh*^p9~*O5H4lyN z8XxAoo0GJbzvq2;{dcZ^Z*r%iIZ@HP9$07jigv!dW8(~;dtpiQIKPN5ZrN!3Bkzx{ z|J&<-IJtB7QsV5T?Wyfo`Lmb#GakN;67s!^!@H&B51qf`8t#et98bqu!^^z-HZu^y-;}`RDOKpRUz-x;>F#JI$Gm4HYz3bd81;sysvM2>zgH5Z zZMUHEz1P-a-+OaCBov%nGVGQ**68(%TgpUf&wb01c15>qDOs)me&>#*Az^7)@BiV@ z4~8~Lp~d(WD~aT`+BGx3=;EJdLEovJ+Vf8?<}sYe_OkCZ=2((Dc%#{tAc`+ z;Z01s(v2fqRf&?z{Pl^1dGZl?<8fZ;YUf&|kk__&5xudoq5)i4zrL;EXJ1X6V-uxZ zY8Z02){kuVCrWy^FDJ~SkI1p7xdc?%Zk#ZhX6O?aBu8Z|-ceJIDJH$NRSK3dcth1*81bjf)pQHWuLSbZ%VT805>&3Fh-Wc|P5p^($MAiGoW&#|gvuBlWe%1(mC@ zwfIKkMmbs%bsnLB0wT1QCrS9-hr*<}G+9`e$m#x6ugTZGH3Y5p?3oCQbvd-+S@zxc zt$Kx=n#I14jd{4$^INLTdA`gon2~XGy48)l+uCPG-L-LhW0H4W;jiA{uiWHs&hXA# zLP-b^0qw6|0T!_N22V}$H^aPVHeroO%_pq|?>JYj?^J!eYRx9(9myUtyx`=^x&-rS zo;;my{Fxz_)Feq$(qLP;zB0_)k8ezGjBj4wHf%v-Q#={?Ort8*LMNV@h{BThY<%U( zO@~l&UMRS*Yb$zB%~zb*R0+jr1l!r&#umPHl)p?1jaPOWClig6{Ee?YYV-@0(VsT@ z*DmwS^bQkFFyS30^6=F}qd!p@mCH~qfe-u{b{2Q$^?Qtf!m3| z?VUgvmGLDiXLoDs`G)@OBBA#3PVLo1?Ny=n8h;~@s15Aeiusab8z%%?*N*K%!gfKh zb#F~4Y{1bf-V5@z-CMmv<)Bb{@v*a+Z-K5}6`bQc&YKD6O~Ls!K6pFfyuIsaNal9# zDT%z062@L<{#2taH+-6BacCEZpVknz`tQg1rh)ASpa~J({bu=gccO4q>Uf4|`nIm_TPN1C=T=Y7LEZV<#i6GLBGUr6uK5i@nh^{ZUOKfgBktw%7de;mPdyA-GZt$&D!gm zO^Jg3`-UaO(kscrig#w$tUCn_iGqgpVxgcN*mY6MdR!>#S~7i7Sd%QNO_tR>Ji9iv zQ`(#;ZC(!xrKgkj`VVr}+jpwk6IJbeN3T%jPF6S`-dI=el(#0zTQ|%?`T69&378N{ zPCZ)@VA^I_B4J)lwv`yXXWKR9eaG=m<8L>vM%Hif6|Zb564vfVCV0DDw$%HHv4}78 z2u4q`*s)F~irbb3;c3KUNkpxjeMDA2Z6=HsdB8t2)bF*y8}zr(6D+>0TQK+VWX~r$ z<2UcUb#H|fbS10fg02oS_ms+<8;Z|VYKviyQ0omq1*9nhMGLwbTm>VTr+ISvaT=-Q zBZK|_1*GD7-e`c`nwd}*mgrIBKEv#4#KD_nrt+h9V%1)v1_)cNhX5Nx< zw+w{O2PfY@$v2%9YR?H}=Xc6_;c*LD3{qtE=(<5DZO3x?!dBT<&Njtgp5VtO`KlX& z^(Jq+`LVH-uWH?B6{=hq%FXv(;oVpHtH9E3^Vje2Rbjz8%bRB5725K>`}bCjI~GU6 z;#lwf;lK|DHYRtPx)V*^xai=vW4kb|MZhbW?AZsEPSCZZM5F1KPvL=tsrWNO1Jd4{ zw^IFJdP$Xp-sTI&@-^Lh$M2ap>Nlw$UHgM;e>fqu4{Q(e*CuzaO(m{P3D<(ca4=1{ z>#3E^Bx zxS4!ZrUm~mh>~!-;V&w*;QvjhA>63?n{zIExK8!A4h8b-G>{~cKKQX4*bVuf;Ikox zsGf$Op}>z&oORNM5?uqqL#G9f{S0F4-=a?w>`>t6C|dC8iu5YdoJx{^ph5ST$e3G7|pv^7*=?ZBZi+8M(C@p}zY2>|* zyhY^QM;=n+(V);a)aMfo)B;>}&ARWwE6uziW??GoXM_0agPos?;RlQONj19%h2c~` z1}BH?4+#8RMA{vYXJ(_p`7m>i?S!h~KfxV?4Lm$dR49H<7=KRae@+;11mgKWCknv- zPeijoH1BDZic?G69s$8Bg@ZNfDf-$o=oR@ZJ$nQMYYrT&cRod5d&N3M(Mrc20l`{6 z4%YLZqOU!hT47s-G7zlwJVn8tnQ$rG3V!Ae|HjvUK@=*66f4ji3Q<`U!Yi!&-_Mca z+XFm(jlX___f7HFrxJ7^F&va~_`3^I1hCMn98^MZQF#dkqe_qT|KKVX78ydM$Nqr^RM%|sz zJ(&r%JAovH70xmel%#;5qy>a8JFKN-5SHVxj#5CF#Nlj831Jlu=TK@&L}}nJYogpDp|p^q zLn%ISSPyYJi-^jGxSV8ME~STGxj3(ZGC(*Fhx4d>2p8aRKHyq7A)<;V#8mNwgesYk zQl%3zs%%0|l}{)rqlF9?P$qz{!0?6CJ_uLha1m7n;r%#Ve5r)0Zaqg3ZXH3C5+%f? z(hA~IaRt$mBxMUw61gc6C8{J$#HBJh;f5Y4C5#w$;>|R~rE*BCL1|&*B@>iW0e?mz z{gT+$&6kzwQwNTU`QOR%L;^gjF>LHGdPQF1xTK8mip({?L!d)uDJSXh&v4RiuPe-v z!@;mO;CK2s8664wJT8QB5^F*ZVF~>G9#{o9!cI(xTfYIVg}74UtO(UH4k^5H~p zI&6_t67NfYhw$X2e&@7@Q@A~o&WJC}$r(>L!ua18agtD&F^f3)WPq7=hC>K}eOzmI z+qf%m-NSgp!nfWT^p1yIj5in#jYHVsVw^X89>x)x4qWp%!k$pLAvnWn91f@79|$|c z9*2X;fzmYa7peenmiPl979INxd2fZb(C6+5{@fqIk-rBZ!j&W|3S$yrGF*ipvqZXS zwoE7tN)Q!5CZQ98V!PN=gotGq+hj?AjE}%32a<6-VeR5df(CYZUo>QvaN@4$}^yl4JP^V}g;+PYEz9-{}~KdJvyQcVnz!ePI@^I z@R>ynj2VLOX^cAVChVd9O7)del=~S0-rHSU=ySgce}3G8BYBilRY5*P4BS`$3IcoW zt4YWSeFe0Cc7NFvQ1}if^kq`NlrHD^o2ZTDU%1;n0(|FpaNvRzzq)r@oCPG zz{p?fcyBjvq0jwX{P{637yNpXj}Xl;AKk~K+2fdpK6%kGeuj)o6*pdvshBlSUxBH}Wlk`Tl@V*miNYzJd7br8!btv6jBEax+22ZMye zcIiQ4-T<;P7D&VNvqE4u)>y2(XMWdw@8Ay(VkX%VOJoI`I{iZPottyIKW?fGj|q{i z0Mc%2wQe%-eszZGy&^SXCxgZO)HY|6w%VKR*AsO#aNn@@>fpWy?K38GuvFNV))@fq|P7E|Ia^otUIj?zZNd_CKxAL>bD zTHqd?1AmsFh@cD%aIr-cl-nhqgGd6U``{^7kVKGRyIDR|Diul=+vSwB8mhF55yv9u zhQVA)P%^t%Lo}guIZ99WlPx6?F*x>Y2(Th@+9_}P+-q7+AyToqKnAWm`9 zuH_>Ew!@8uDN-=OkBb~2lOqyGs`MafCv8ZF@H2xlO2}<^62h)bQ3)6i zdIDJh*x8i=&CiG^q8V_Bh$-kjifBQ-3C;)_!$cg!k^%j`0z$Nu@QRHf?2Y&z1rYp% z0rXCq9>|c`;cOY>R7e7G7DOcmRkX@hD#WJ0Da=g>3Sho6b)bdHppt$nE$D5u2&%`8 z0u~g)4eH{JT%*C zQ$cO=-D(pwyg6%#lXtj~CXZ%2gF&Cy+`rxWkMM79vaWW6X1Q{ivyr3JApULE zV9c50fYVN){1S@mK*&k}D^zhPh)o_&1XIwUa|UZmB)Y4stE#F@yyNw$$}trV2eDFR zE&>D=&=C=mg$cE8w);~uo7GosNyIw&Z10wwAT=K=!1_sR*9~3ux~_WZMoed!lP31H z%qbSM>!k4$g(k7TZh3UMm#su`x^>d@NR>U`aI0ZMWm;F6mU34M{=W2YOaHE%RheR{ zbF*DrQl)G_w3(GVKXGefBdc;ft8%G}&8pnUIu_45wn``T`R|y&WB%U34Sm(RzG`VK zrmvstO5_&2bNoBUKd6kG4y}N9WUYnG?frv{(6&KF_#ewU_T^(aq0!H`-)i4bSFEcm z9upFoO_ZoT_>-!iR4rG{Ip-&DP2QQB58MhYUWsd~S32h;pVV36dO#qquUVnz6r0-O z#nGih?;ZKkk)@jDishc=)|EpmBP-U`oYkYNX11g|rnNq3VaeeyH`O_ORi<^d3E76w zD9G(twlDXsbgZ~mMpr9VJ64<7l2bA5OKWbHJpCmyNd`n;godFDetx*7tDX3;tb^#P zk$qIAf*&8%sJo6SKWdhu@G%MU+l{><$uCOUdSsGc=3Bu3mCOV&zp54?|3F?(yX05x zG6-)oV30|G)!kJH>~WnVWrc#Nl2j$2s8X?MHM!kDu}IMb5StbmES$C`B)m0I2M9ga zeCP$B5Hd-S`Jlq)1M-pifWF9lK;NHM^536Delhav%C&QU{q@gq{C4#ZOpcchFg<<^ z?U*IfZP-oK@21}EpnfQkMPw(QHC3j>N0VwPD0!X*Z9~@m4mZ|BNuHs6rFQ9VCP7O1 z?qU{DQc6b2g))OOC;=I_JDgpXRILgM{_R1vDl$}Ss!UsUDwqn`&{E6}2P(vkRfwFH z?`9UHR4b?-P_6Pmt?i&*$)GH4T2M+TaTg;16r~EQF%?#tDq2RVGmY@pw&pMkU~RD7fS6$9-0le4S65KC>dgU;D=eu^rM8+$QwZ3Ao9*2ZwPrvb}=^O zokbpnyb`g(tl)Ln0Oh zMy#kdx9gSVo2z+j!>I@Izt{Y&=I^vvy^Mi+5tDmUFDvIk^TqV1QaWZy!sb>!PqT<& z{*Y)t@j>?o-K*W(HOsPhE)dtYu3es!JWHK4uZI8n*y=&Hq$j5BeL%D1**{68^Z~Jc zTGLre{JhlCnJ@cLt%C4}`RdLC$`31~D11PI{92>6Qu2|qqrCfoHr9 zYq{i?}^ON>~JuhNXAkGQc!}ki)=}G zF2ta38afje+9%>f7PA;O0r09eC-?EI+hXQ2X)!n7U-q^-x{0)cxMTj|NsGv8tAb>K0bol3u_Onv-g1BQD~_*DNv? zoA|-1fdZb}CukgpkWk|nnHtctQp4MznBj95)*#`N#u31-kppyaqC!ya%Fy(lm(;i~ULkVr}yziE8Zep<+0loX$qMudQv(ox>_X#Y9 zR)X+YLVDr_BmrV}?9z5*rY1#<_kMDfQwG$Ec=&ZE3BS;U*PYM^dzNT$KzgVgJj8}_ zwAKk8S`mlO-PZeE|MCQmNw)>#1D+KAa9fa`Lv7)=O3-2fSZ+d#Gq%3aGDF%52HOSj zL(A4>AP;sfT8sHrGBn8obE;h8d+L3XABpx!IQ8hzh|Sv7-_vjH=HzEbt)uW&P@_Xb{X@N+ zvTOK^ZO}@iw4RRsL6p|fIZWXQTgT`KL`a6LW81`jia;+YPg0$Yx|Ly2)5Uy8B7^f; zeHe!Hlqk`Q@;)QLyAOQ`pC32lUr+X5N-KQ`eU;IFqRMT(m+rBaB!a@==a(M~L|qW9yvx`Y zhAl!5wq-MLRtRwEFKio%?UJ4JM5V&kA*UD}8WInk!!;D(`T-O%(8hL*YRgf9E7nZ=HKr{6ujoU`1rT)h6+QYGjH z3Y{?$!?_#@oXe5Ixg6Pq)FOiobtHRH5@RiyIuphpvZ1k~(PC3%mXYC$EQeV~8@1Uvv-d-3eiZN?v-|+B2v`R2lx`ypm zs-I4dtR$R+K5w(OnP6oHjVDgw4Y|Si2=fYV5LHHl2g&KNgoY8Pn1%&ium1`sx#pP( zAsWJdy)enkbzmFF0f#@XT!FJLVOZK_z6&szDZUK>7*R^?e92tN!oCf%WSuODk!6^d zzd7^9%r{?s^VLm_Zb5XXXHJw*YZgS`KE81FTgMZ*hPl2i385)~v7pRfcqKt*FH|JR z+=a6V(y-u6kU0y7w$yUG#;GOLn)w5_4%{*0qXFOe8tm+T>ot}v*;M7TMJHB0t7B|| zHKyuer9EjT6b1-VCYTAUFtX3Kwk=4&iTR~5WL(3rc%EHz7r;XKxa>a=uSi5$pJD$k z7ysoEg5;D#uo$85LqCCOEg*_EOGJWh3TDQOVPht#6G7Lc;bFxn6)#TfPaBd6J6r{v zbrZA+Iwq1DIe-IbA<472T!6g-f&w1o>4N-81qsYuL0{TVSBne6~ z+NAmu%ip;IBu-H%V>|aL6w;(9B+I}ki_-o{Eq*#i@{C$^RCWqeMMiuMrBB6^<0TB~@uxQB`)eMFA3gzp#!f zP{c^sH3A>I{{m|Zbi@R^^tM>qPNJ$)_-2n+2$0kPd-jBisu9+XZ@{Pw<^=WQNRVP@ z*N-DPgPl-9s$PDq>1{%U00>{oEMQ92w%!0u#ZP$60tj``uHQc4-2=7is9LJto^267 zg)&(JLT*?N1U-rj0_>o@ie1{9YtNZ@(5eR;7Hn#e9ywG)>(2mZh=JT`oGtbo7~zd} z1z;`}+BYTI1LlXI?0UQ6X_$|v`Vjb`NuYPo0Q16hpEcVJm{(e0XW+114Ci={Jkfv9 zGrW5)=N$!%TJ8GhV?-0wODT*BGV!4cC?hSQT4<3)`~)tAbm5a$l7-lnXF4pYQMGG& zr$F;y5nUwoadIgo-Ai3`;aBFYZaRmu?g3+~fYA%luO~&nV(Kt0g(Dy(w9EtZN!VkM z(Ir%GHFVlG$`;c_5Z@;ptVoW}Qo8hoY1%Jf7%Zd9o=UURbm>>7*#OXNkS=>63Y?+K zU<3`N=d{7{ZS5_mi-0n6YM6#G@AE)+?Zy z4xK5=;1_h#VTY?A(g!w;z%2uy_vpq+SiRUbn^_RrFeRcJceS@uJOMSs3cq`23E$a6 z3E$ndgzu$G$Y|5E$74fs4DhqF^t|Z*j{$=E=Qsx5#bW^Eb}6xE&IhskT;BdR=FGpr zqvM1yl%A3+CC>!DeNw(o5Wyqk&nqV-;htH}QCcfl4(=i>1CG#I#QjgKOuS7W0qy%Q zXf0S187mXCZb$y3v~Jh@-=~k_{JI_acb5w(nVBjb*0*2|yE}ABRtnHh%-G+d8@F3Q z6R;Ebl+w~N=?9oEu>Cy&dma<8)D8ROH9`-VS&J*Yz%%_gg`KW{| zO$dr=Oa9{5gpbqtwr5z#IA1=_*ZzD+H*n#e^uh_Mm^w-4Kh4sCeg9Lzyi<8ra*&-$ zhSPVG(LO4ge|W(cLgpWG24Rn-!`AQU>=>~+;Cx856rIh66C!Z5qO9Wlg}2xSVX-PVyVs^3QU4-at#LL%Kk^;&61hs}m_N7Z2`1KYK51v*&& z1l04W0|)(m;VCYEN8zxAH3F4JWsMDu4TrcgA=x%OIOymfqOFv@1GbH=UBg4&Bb=y- zE8PY`_n)y2kJ2ds$&toJZodEmgdea{4qJcMz^Ki!6IR^Zm`MmLY`b+0Kx_LQeb$aa zx{ux^HAxcS0MFjl0;J-LnHl&z$5s48QeCQhARzl5r7F? z^}t2=ES|cZaM*5?@o~AqfX~+tS1B0S4IhEq3a}T>=}{!j-+&Y1ih-f7c^I45buALy zo`#N)di`#=jS*E?k{uv{E@e7_3-Fmxxbk}~x*zczpA09PT-QtyzvpHUaDw|5=+Z*K zAM&)BI3+M$AMQnb)(M|MjaJi`Crs#!AUZr{f=lm^$?2MMUiNuRHT?EFoTxHG2?oB2 zHpYYN8Nqiiq8f`6?uJ-G-Yfn{kdxYmM`(@=CPxb1>4y!tqku7|;bHiEa1IOSh?4lS zNK4}}PKHMZTo`ns3xf)$58d^c;j)t1u6f}q2;BXESh%n<6jk#T`vPz}i<9G9h9OSv zgbOaA5RL8}M$LmL%@m42oW~91xEu+aG5}1|-VmI0xB_ic<6@GiNh2W-6IEK0eG8?! zJ^Z5qa9swr7Ll#SH%aVuM+>?Dqp-)6gic;{fiiPlIKjqqNV05Y60F}HRi!GhL=Beo z=oBvIgg6E27htF8zGNLdkSXK#PP)1>zB;}~JJR=ik`>D!6I7bFTSXRVEWDEtg62V` z$xy0c$=~Ukk`1%-IfRBK!EnQp;gBV&v*0Hd>O;^gxOnBX^Crpy_tXo8fySRMIn!}Y z7=q%%jT{Mp-EdiHDlM$iaNSlL-N%WVIlb5K@;9}SNY5CL$v$kpv}!GRe>x|ICmE4B`G zbPifUy2051y1$D-OWh1I-Z>J+tJ?=8NlR2d3&^hSLF>o}r%m3J739Q@qMwk4M;bX@ z@-d4MPdE%*9pZ}Mp$ypX2j+#-Z?3TSy2s&!55m;Ao%G$-9+0%4MrdlZi|)5uIYVYH zI8=c2UWnVbGg~TyW59YIY08o4_hBf(72i7en9vsuUmS3IeDDrM9pjmXM>*;u{!rvH z8daA)buM(&$XAEO3d#bvtLlOg2Bc8PQwJ5pF(iDb-o%GNs(He7ZV*$@Cg7wx==N!^ zABEgZAXw*_4u)q;E+-SNbNS!~8xC^C+i}l4alza{ouW=2kU&@8I5j3F+=iRo@j-_2!t{ob$_lmQl-T+X(iL-4LiG<(NTu|(yT5RCXR zC$WvvoF0d_SCWO}sHBwd0ZEHw6Q_i?8?03z(^3cxKmP8VjJ{t#dh@beKGd_&<3 z;O)HP;gq;AkGmtxsU`uv&=d}9G8@9oARfK}UUYvq9JuB6h06oHW!g?1w;BZl5B7Kr`fknylOQ0yw^Ow8C(_U(NF}s z`bvYz3L(a03Qr{~h9dY>2tW-?#P0_ zTvLp|p)*`Qyo6zL`rS~rGi>sPLng}F-9LhiM;zVVL3arDk|_v>%EP8`03#BDZiHqs z*S(;GOwJIF1ohm&ybctLzzXvYQusX}DNUf^Jq)NPo*tIuK~hWt+$AQsl8>7|+XMRG>V#rU5w{T?7amRq1?lMzy%GZf~ODB9z8fJqF`RpE?KD;E8@22C=I z&&qOS$nOk>rUGG3jpC6ULh?F0-D`YIs1@i4=lZ^qOm;uLvN?l==|EI zTE}Krvo$?2b?elhRwY4yEm4?KYV@J8_R2*Q*9PkEsZXpUK7WQyKd>`%ES$wx0Fx;9P`Xs=z6Pt@yOE1-4nRL z-Usrvn-4$%2Gzrvnm;>Dq4%Tky&ELX48toE`+-4D*jwZn1J z_Bfk>0;`s?V%ZI|{RrvnOEasAwY&!^mL6x1UWywg-Z(ScGiOa0iryJm7+9n>a;w&J ztCr3!GqK#Jjoc&axkpxBkL6lt2R~64!0jBitY%oowRYBiIiBx&V;FGkP3Vi>sb8oE9CNDH zbE=l;SWfNi=}**o7_F9-v6X(dos0#+B-YT3LqtbHu5Iv1Cof0S3g z7+#95)UK4WFqHOFvAhxBUwvtUgvYn?6WZd0*02y*npv^1S)GrSvRwHa1As!;mWI&j z=e={@1!~@L%dtquvi8mPe4@_5D4bf8tVY-pYfOty&}X1NoF$KE;IJDRiuNT)O+sZ@xUewDnhvi_uAE!F@IbW&bzWvk*Y9PL zVmaUftyUV|9=U5^%Uf5gVnwH7c`qeM{o7r4rEJ-eRY|O{J!UwOAWMH-#af2hv!k(E z`$p~fdhIxS>E(E>Ggcm0uXQe+Wj&J{9^bmhx8d>Mdu8zzh+HoZr1Oo$YUz#I3+uHP z*ozbKT1TvWdcD>G`CRS|*VT2`)eV=A&*xY#pH7e!_gmPS-UkJ-%Cj4lW9yY;vC8x8 zCD(eT3t`vI79CnS5;L@I7+zX8yc9EZu1&5R1`}lA-O9Uewz6}rJ67HwD?XhdixQdx zKQ^!pfcsd?d~U-$v2LD-nO|nzSJ%x~6PkVZyA#f-oAoq4wJQGU^4 z&He88`v0>3Uk|J_Jgj;tmftZuj2{I_WaoiYQ)$r~dUp;k>K>{NY~~r4WJ|BE*2VLB zZ>i=)b4L^TW$#RbaMEw&)vV{$EEmS|nm|JoG%QDB1#NShPc?=GUqVx`Se(!lE*`yi zV)4Y%<&EO{_2T+vU#$3eLQ}S+crR-yYq@E|*tBkJVw<~TMr%S-yf}965?odkZ%jEQ%%m7TWTo$fB(-(5n&99{uft19YTxS|GBQ+ zcZzsuDh!CU4^PO!|A)%LfLy;p$ie?_r_}zFlK-yisPP|@{Et=<@{h?NiX$EH-X6@) z9I)6gorE`%%$5Aa7M;6?XKiOf%$E>i9;2Tcm?^GR77Bt4~u4Ml?a2J+B6nV&JE=>tL?O@AE@ zPBk(EdtKAkn{d54XTURaIpAd6c=Lma1jG2zLHzy%^G8$ynuNpZe;8KvnD+^Mw}zjZ zknPg}cf{v8$+SWF@Sg}3fe*KEMWWvl>faK|-x4YufN<__i9GN>A?jm9{pWIchWB$7 zyzlyXAw0+VxdGl@+|r^q@7ARJV|MJ7YstWCU*q4%+iF6O+N_HCCvQG4hGz{vC*dCb z=UTYT3}Q?spM5v7`yE5K4>x2=}HmW#}!Va!6dvy%T_&gbDQWr5M1; ztXnR`z>CfI+TLw@NLC}5oIBHSXX?u>89;o2@AG|Fl7-)y$nC0@e0WlX{8Rh7DkL9O I$RPZG08C61G5`Po literal 0 HcmV?d00001 diff --git a/scripts/api_crawler_smoke_test.py b/scripts/api_crawler_smoke_test.py new file mode 100644 index 0000000..fd5c618 --- /dev/null +++ b/scripts/api_crawler_smoke_test.py @@ -0,0 +1,411 @@ +# Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +# Proprietary and confidential. Unauthorized use is strictly prohibited. +# See LICENSE file in the project root for full license information. + +import json +import os +import sys +import time +import uuid +import urllib.error +import urllib.request +import urllib.parse +from typing import Any, Dict, Optional, Tuple + + +def _env(name: str, default: str) -> str: + v = os.getenv(name) + return v if v is not None and v.strip() else default + + +def _env_bool(name: str, default: bool) -> bool: + raw = os.getenv(name) + if raw is None: + return default + v = raw.strip().lower() + if v in {"1", "true", "yes", "y", "on"}: + return True + if v in {"0", "false", "no", "n", "off"}: + return False + return default + + +def _env_int(name: str, default: int) -> int: + raw = os.getenv(name) + if raw is None or not raw.strip(): + return default + try: + return int(raw) + except Exception: + return default + + +def _json_loads_maybe(text: str) -> Any: + try: + return json.loads(text) + except Exception: + return text + + +def _http_json(method: str, url: str, *, token: Optional[str] = None, body: Optional[Dict[str, Any]] = None, timeout: int = 30) -> Tuple[int, Any]: + data = None + headers = { + "Accept": "application/json", + } + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + if token: + headers["Authorization"] = f"Bearer {token}" + + req = urllib.request.Request(url, data=data, headers=headers, method=method.upper()) + + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8") + return resp.status, _json_loads_maybe(raw) if raw else None + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8") if e.fp else "" + payload = _json_loads_maybe(raw) if raw else None + raise RuntimeError(f"HTTP {e.code} {method} {url} {payload}") from None + except urllib.error.URLError as e: + raise RuntimeError(f"Request failed {method} {url} {e}") from None + + +def _http_text(method: str, url: str, *, timeout: int = 15) -> Tuple[int, str]: + req = urllib.request.Request(url, headers={"Accept": "*/*"}, method=method.upper()) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8", errors="replace") + return resp.status, raw + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8", errors="replace") if e.fp else "" + raise RuntimeError(f"HTTP {e.code} {method} {url} {raw[:500]}") from None + except urllib.error.URLError as e: + raise RuntimeError(f"Request failed {method} {url} {e}") from None + + +def _join_url(base: str, path: str) -> str: + return base.rstrip("/") + "/" + path.lstrip("/") + + +def _origin(url: str) -> str: + parts = urllib.parse.urlsplit(url) + if not parts.scheme or not parts.netloc: + raise RuntimeError(f"Invalid URL: {url}") + return f"{parts.scheme}://{parts.netloc}" + + +def _as_int(value: Any) -> Optional[int]: + if value is None: + return None + try: + return int(value) + except Exception: + return None + + +_CRAWL_STATUS_BY_NUMBER = { + 0: "UNSPECIFIED", + 1: "QUEUED", + 2: "RUNNING", + 3: "COMPLETED", + 4: "FAILED", + 5: "ABORTED", + 6: "PAUSED", + 7: "NEW", +} + + +def _normalize_crawl_status(value: Any) -> str: + if value is None: + return "UNKNOWN" + + if isinstance(value, int): + return _CRAWL_STATUS_BY_NUMBER.get(value, f"UNKNOWN({value})") + + if isinstance(value, str): + v = value.strip().upper() + for name in _CRAWL_STATUS_BY_NUMBER.values(): + if v == name or v.endswith(f"_{name}"): + return name + return v or "UNKNOWN" + + n = _as_int(value) + if n is not None: + return _CRAWL_STATUS_BY_NUMBER.get(n, f"UNKNOWN({n})") + + return "UNKNOWN" + + +def _parse_trigger_type(value: str) -> int: + v = (value or "").strip() + if v.isdigit(): + n = int(v) + if 0 <= n <= 4: + return n + mapping = { + "UNSPECIFIED": 0, + "MANUAL": 1, + "SCHEDULED": 2, + "CI_TRIGGER": 3, + "WEBHOOK": 4, + } + key = v.upper() + if key in mapping: + return mapping[key] + raise RuntimeError(f"Invalid COVERIT_CRAWL_TRIGGER_TYPE: {value}") + + +def main() -> int: + base = _env("COVERIT_API_BASE_URL", "http://localhost:3000/api/v1") + password = _env("COVERIT_TEST_PASSWORD", "TestPassword123!@#") + name = _env("COVERIT_TEST_NAME", "API Smoke Tester") + email = os.getenv("COVERIT_TEST_EMAIL") + if not email or not email.strip(): + email = f"api-smoke-{uuid.uuid4().hex[:12]}@example.com" + + project_name = _env("COVERIT_PROJECT_NAME", f"api-smoke-{uuid.uuid4().hex[:8]}") + project_description = os.getenv("COVERIT_PROJECT_DESCRIPTION") + + target_app_name = _env("COVERIT_TARGET_APP_NAME", f"target-app-{uuid.uuid4().hex[:8]}") + target_base_url = _env("COVERIT_TARGET_BASE_URL", "http://localhost:3000/health") + target_version = _env("COVERIT_TARGET_VERSION", "0.0.1") + + poll_interval = float(_env("COVERIT_POLL_INTERVAL_SECONDS", "2")) + poll_timeout = int(_env("COVERIT_POLL_TIMEOUT_SECONDS", "600")) + + pickup_timeout = float(_env("COVERIT_WORKER_PICKUP_TIMEOUT_SECONDS", "30")) + precheck_api = _env_bool("COVERIT_PRECHECK_API_HEALTH", True) + precheck_target = _env_bool("COVERIT_PRECHECK_TARGET_URL", True) + min_states = _env_int("COVERIT_ASSERT_MIN_STATES", 0) + min_transitions = _env_int("COVERIT_ASSERT_MIN_TRANSITIONS", 0) + + print( + json.dumps( + { + "base": base, + "email": email, + "project": project_name, + "targetBaseUrl": target_base_url, + "pollIntervalSeconds": poll_interval, + "pollTimeoutSeconds": poll_timeout, + "workerPickupTimeoutSeconds": pickup_timeout, + }, + indent=2, + ) + ) + + if precheck_api: + health_url = _join_url(_origin(base), "/health") + status, payload = _http_json("GET", health_url, timeout=10) + if not isinstance(payload, dict) or payload.get("status") != "ok": + raise RuntimeError(f"API healthcheck unexpected response: {payload}") + print(json.dumps({"apiHealth": "ok", "url": health_url}, indent=2)) + + if precheck_target: + status, _ = _http_text("GET", target_base_url, timeout=10) + if status < 200 or status >= 500: + raise RuntimeError(f"Target base URL not reachable (status {status}): {target_base_url}") + print(json.dumps({"targetPrecheckStatus": status, "url": target_base_url}, indent=2)) + + signup_url = _join_url(base, "/auth/signup") + try: + status, payload = _http_json("POST", signup_url, body={"email": email, "password": password, "name": name}) + print(json.dumps({"signupStatus": status, "signup": payload}, indent=2)) + except RuntimeError as e: + msg = str(e) + if "HTTP 409" in msg or "EMAIL" in msg or "taken" in msg.lower() or "already" in msg.lower(): + print(json.dumps({"signupSkipped": True, "reason": msg}, indent=2)) + else: + raise + + login_url = _join_url(base, "/auth/login") + status, login = _http_json("POST", login_url, body={"email": email, "password": password}) + access_token = (login or {}).get("tokens", {}).get("accessToken") + if not access_token: + raise RuntimeError(f"Login succeeded but accessToken missing: {login}") + print(json.dumps({"loginStatus": status, "user": (login or {}).get("user")}, indent=2)) + + create_project_url = _join_url(base, "/projects") + project_body: Dict[str, Any] = {"name": project_name} + if project_description is not None: + project_body["description"] = project_description + status, project = _http_json("POST", create_project_url, token=access_token, body=project_body) + project_id = (project or {}).get("id") + if not project_id: + raise RuntimeError(f"Create project failed: {project}") + print(json.dumps({"createProjectStatus": status, "projectId": project_id}, indent=2)) + + create_app_url = _join_url(base, f"/projects/{project_id}/target-applications") + status, app = _http_json( + "POST", + create_app_url, + token=access_token, + body={"name": target_app_name, "baseUrl": target_base_url}, + ) + app_id = (app or {}).get("id") + if not app_id: + raise RuntimeError(f"Create target application failed: {app}") + print(json.dumps({"createTargetApplicationStatus": status, "appId": app_id}, indent=2)) + + create_version_url = _join_url(base, f"/projects/{project_id}/target-applications/{app_id}/versions") + status, ver = _http_json("POST", create_version_url, token=access_token, body={"version": target_version}) + version_id = (ver or {}).get("id") + if not version_id: + raise RuntimeError(f"Create version failed: {ver}") + print(json.dumps({"createVersionStatus": status, "versionId": version_id}, indent=2)) + + create_session_url = _join_url( + base, + f"/projects/{project_id}/target-applications/{app_id}/versions/{version_id}/crawl-sessions", + ) + + crawl_config: Dict[str, Any] = { + "maxStates": int(_env("COVERIT_CRAWL_MAX_STATES", "50")), + "maxDepth": int(_env("COVERIT_CRAWL_MAX_DEPTH", "3")), + "includeUrlPatterns": [p for p in _env("COVERIT_CRAWL_INCLUDE_PATTERNS", ".*").split(",") if p.strip()], + "excludeUrlPatterns": [p for p in _env("COVERIT_CRAWL_EXCLUDE_PATTERNS", "").split(",") if p.strip()], + "enableSemanticDecisions": _env("COVERIT_CRAWL_ENABLE_SEMANTIC", "false").lower() == "true", + "headless": _env("COVERIT_CRAWL_HEADLESS", "true").lower() == "true", + "timeoutSeconds": int(_env("COVERIT_CRAWL_TIMEOUT_SECONDS", "60")), + "crawlerSettings": { + "defer_destructive_actions": _env("COVERIT_CRAWL_DEFER_DESTRUCTIVE", "true").lower() == "true", + "destructive_keywords": _env( + "COVERIT_CRAWL_DESTRUCTIVE_KEYWORDS", + "logout,log out,sign out,delete,remove,unsubscribe,cancel,checkout,pay,purchase,order,place order,reset,deactivate,terminate,drop,empty cart,clear cart", + ), + }, + } + + trigger_type = _parse_trigger_type(_env("COVERIT_CRAWL_TRIGGER_TYPE", "MANUAL")) + + status, session = _http_json( + "POST", + create_session_url, + token=access_token, + body={ + "triggerType": trigger_type, + "crawlConfig": crawl_config, + }, + ) + + session_id = (session or {}).get("id") + if not session_id: + raise RuntimeError(f"Create crawl session failed: {session}") + + initial_status_raw = (session or {}).get("status") + print( + json.dumps( + { + "createSessionStatus": status, + "crawlSessionId": session_id, + "initialStatus": initial_status_raw, + "initialStatusName": _normalize_crawl_status(initial_status_raw), + }, + indent=2, + ) + ) + + start_url = _join_url( + base, + f"/projects/{project_id}/target-applications/{app_id}/versions/{version_id}/crawl-sessions/{session_id}/start", + ) + status, started = _http_json("PUT", start_url, token=access_token, body={}) + print(json.dumps({"startSessionStatus": status, "startResponse": started}, indent=2)) + + details_url = _join_url( + base, + f"/projects/{project_id}/target-applications/{app_id}/versions/{version_id}/crawl-sessions/{session_id}", + ) + + deadline = time.time() + poll_timeout + + pickup_deadline = min(deadline, time.time() + pickup_timeout) + last = None + picked_up = False + while time.time() < pickup_deadline: + _, details = _http_json("GET", details_url, token=access_token) + status_raw = (details or {}).get("status") + status_value = _normalize_crawl_status(status_raw) + snapshot = { + "status": status_value, + "rawStatus": status_raw, + "stateCount": (details or {}).get("stateCount"), + "transitionCount": (details or {}).get("transitionCount"), + "errorMessage": (details or {}).get("errorMessage"), + "startedAt": (details or {}).get("startedAt"), + "finishedAt": (details or {}).get("finishedAt"), + } + + if snapshot != last: + print(json.dumps({"crawlSession": snapshot}, indent=2)) + last = snapshot + + if status_value in {"RUNNING", "COMPLETED", "FAILED", "ABORTED", "PAUSED"}: + picked_up = status_value != "QUEUED" + break + + time.sleep(poll_interval) + + if last is None: + raise RuntimeError("Did not receive crawl session details") + + if last.get("status") == "QUEUED": + raise RuntimeError( + "Crawl session is still QUEUED after pickup timeout — worker likely not consuming. " + "Ensure the crawler consumer is running (coverit-crawler: python -m src.workers.queue_consumer) " + "and that its REDIS_URL and DATABASE_URL point to the same services as the API." + ) + + if last.get("status") == "NEW": + raise RuntimeError("Crawl session never entered QUEUED/RUNNING — start may not have worked") + + while time.time() < deadline: + _, details = _http_json("GET", details_url, token=access_token) + status_raw = (details or {}).get("status") + status_value = _normalize_crawl_status(status_raw) + snapshot = { + "status": status_value, + "rawStatus": status_raw, + "stateCount": (details or {}).get("stateCount"), + "transitionCount": (details or {}).get("transitionCount"), + "errorMessage": (details or {}).get("errorMessage"), + "startedAt": (details or {}).get("startedAt"), + "finishedAt": (details or {}).get("finishedAt"), + } + + if snapshot != last: + print(json.dumps({"crawlSession": snapshot}, indent=2)) + last = snapshot + + if status_value in {"COMPLETED", "FAILED", "ABORTED", "PAUSED"}: + break + + time.sleep(poll_interval) + + if last.get("status") == "COMPLETED": + state_count = _as_int(last.get("stateCount")) or 0 + transition_count = _as_int(last.get("transitionCount")) or 0 + if min_states and state_count < min_states: + raise RuntimeError(f"Crawl completed but stateCount={state_count} < {min_states}") + if min_transitions and transition_count < min_transitions: + raise RuntimeError(f"Crawl completed but transitionCount={transition_count} < {min_transitions}") + if not last.get("startedAt") or not last.get("finishedAt"): + raise RuntimeError(f"Crawl completed but timestamps missing: {last}") + return 0 + + if last.get("status") in {"FAILED", "ABORTED"}: + return 2 + + return 1 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except KeyboardInterrupt: + raise SystemExit(130) + except Exception as e: + print(str(e), file=sys.stderr) + raise SystemExit(1) diff --git a/src/queues/crawl.queue.ts b/src/queues/crawl.queue.ts index 0e42ff2..bc20beb 100644 --- a/src/queues/crawl.queue.ts +++ b/src/queues/crawl.queue.ts @@ -2,23 +2,13 @@ // Proprietary and confidential. Unauthorized use is strictly prohibited. // See LICENSE file in the project root for full license information. -import { Queue } from 'bullmq'; -import redis from '@lib/redis'; +import { enqueueCrawlSession, markCrawlSessionCancelled } from "@queues/stream/crawlStream"; -export const crawlQueue = new Queue('crawler', { - connection: redis -}); - -export async function addCrawlJob(sessionId: string) { - await crawlQueue.add('crawl', { sessionId }); +export async function addCrawlJob(sessionId: string): Promise { + await enqueueCrawlSession(sessionId); } -export async function removeCrawlJob(sessionId: string) { - const jobs = await crawlQueue.getJobs(['waiting', 'delayed']); - const job = jobs.find((j) => j.data.sessionId === sessionId); - if (job) { - await job.remove(); - return true; - } - return false; +export async function removeCrawlJob(sessionId: string): Promise { + await markCrawlSessionCancelled(sessionId); + return true; } \ No newline at end of file diff --git a/src/queues/stream/crawlStream.ts b/src/queues/stream/crawlStream.ts new file mode 100644 index 0000000..6e29dac --- /dev/null +++ b/src/queues/stream/crawlStream.ts @@ -0,0 +1,32 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + +import redis from "@lib/redis"; + +export const crawlStreamConfig = { + streamKey: "crawl:jobs", + cancelPrefix: "crawl:cancelled:", + maxLen: Number(process.env.CRAWL_STREAM_MAXLEN ?? "10000"), + cancelTtlSeconds: Number(process.env.CRAWL_CANCEL_TTL_SECONDS ?? "86400"), +} as const; + +export async function enqueueCrawlSession(sessionId: string): Promise { + const maxLen = Number.isFinite(crawlStreamConfig.maxLen) ? crawlStreamConfig.maxLen : 10000; + await redis.xadd( + crawlStreamConfig.streamKey, + "MAXLEN", + "~", + String(maxLen), + "*", + "sessionId", + sessionId, + "name", + "crawl", + ); +} + +export async function markCrawlSessionCancelled(sessionId: string): Promise { + const ttl = Number.isFinite(crawlStreamConfig.cancelTtlSeconds) ? crawlStreamConfig.cancelTtlSeconds : 86400; + await redis.set(`${crawlStreamConfig.cancelPrefix}${sessionId}`, "1", "EX", ttl); +} diff --git a/src/services/crawlSession.service.ts b/src/services/crawlSession.service.ts index e8a3c02..8df5da3 100644 --- a/src/services/crawlSession.service.ts +++ b/src/services/crawlSession.service.ts @@ -20,7 +20,6 @@ import { type GetSessionsQuery, } from "@models/crawlSession"; import { removeCrawlJob, addCrawlJob } from "@queues/crawl.queue"; -import { CrawlerJobPayload } from "types/crawler"; import { toIso } from "@utils/date"; @@ -167,14 +166,14 @@ export async function abortSession(sessionId: string): Promise { throw new Error(`Cannot abort session with status ${session.status}`); } - if (session.status === PrismaCrawlStatus.QUEUED) { - await removeCrawlJob(sessionId); - } - await prisma.crawlSession.update({ where: { id: sessionId }, data: { status: PrismaCrawlStatus.ABORTED } }); + + if (session.status === PrismaCrawlStatus.QUEUED) { + await removeCrawlJob(sessionId); + } }; export async function pauseSession(sessionId: string): Promise { @@ -191,139 +190,3 @@ export async function pauseSession(sessionId: string): Promise { data: { status: PrismaCrawlStatus.PAUSED } }); }; - -export async function markQueuedSessionRunning(sessionId: string): Promise { - const result = await prisma.crawlSession.updateMany({ - where: { - id: sessionId, - status: PrismaCrawlStatus.QUEUED, - }, - data: { - status: PrismaCrawlStatus.RUNNING, - startedAt: new Date(), - }, - }); - - if (result.count === 1) return; - - const session = await prisma.crawlSession.findUniqueOrThrow({ - where: { id: sessionId }, - }); - throw new Error(`Cannot start session with status ${session.status}`); -} - -export async function markSessionCompleted(sessionId: string): Promise { - await prisma.crawlSession.update({ - where: { id: sessionId }, - data: { - status: PrismaCrawlStatus.COMPLETED, - finishedAt: new Date(), - error: null, - }, - }); -} - -export async function markSessionFailed(sessionId: string, errorMessage: string): Promise { - await prisma.crawlSession.update({ - where: { id: sessionId }, - data: { - status: PrismaCrawlStatus.FAILED, - finishedAt: new Date(), - error: errorMessage, - }, - }); -} - -export async function isSessionAborted(sessionId: string): Promise { - const session = await prisma.crawlSession.findUnique({ - where: { id: sessionId }, - select: { status: true }, - }); - return session?.status === PrismaCrawlStatus.ABORTED; -} - -export async function markSessionCompletedIfRunning(sessionId: string): Promise { - const result = await prisma.crawlSession.updateMany({ - where: { - id: sessionId, - status: PrismaCrawlStatus.RUNNING, - }, - data: { - status: PrismaCrawlStatus.COMPLETED, - finishedAt: new Date(), - error: null, - }, - }); - return result.count === 1; -} - -export async function markSessionFailedIfRunning(sessionId: string, errorMessage: string): Promise { - const result = await prisma.crawlSession.updateMany({ - where: { - id: sessionId, - status: PrismaCrawlStatus.RUNNING, - }, - data: { - status: PrismaCrawlStatus.FAILED, - finishedAt: new Date(), - error: errorMessage, - }, - }); - return result.count === 1; -} - -export async function markSessionFinishedAtIfAborted(sessionId: string): Promise { - await prisma.crawlSession.updateMany({ - where: { - id: sessionId, - status: PrismaCrawlStatus.ABORTED, - finishedAt: null, - }, - data: { - finishedAt: new Date(), - }, - }); -} - -export async function getCrawlerJobPayload(sessionId: string): Promise { - const session = await prisma.crawlSession.findUniqueOrThrow({ - where: { id: sessionId }, - include: { - appVersion: { - include: { - targetApplication: true, - }, - }, - }, - }); - - const parsed = CrawlConfigSchema.safeParse(session.config); - const crawlConfig = parsed.success ? parsed.data : undefined; - - const settings = { - headless: crawlConfig?.crawlerSettings?.headless ?? crawlConfig?.headless, - timeout_ms: crawlConfig?.crawlerSettings?.timeout_ms ?? (crawlConfig?.timeoutSeconds ? crawlConfig.timeoutSeconds * 1000 : undefined), - max_states: crawlConfig?.crawlerSettings?.max_states ?? crawlConfig?.maxStates, - max_transitions: crawlConfig?.crawlerSettings?.max_transitions, - max_elements_per_state: crawlConfig?.crawlerSettings?.max_elements_per_state, - max_select_options_per_element: crawlConfig?.crawlerSettings?.max_select_options_per_element, - max_action_repeats_per_url: crawlConfig?.crawlerSettings?.max_action_repeats_per_url, - action_retry_count: crawlConfig?.crawlerSettings?.action_retry_count, - replay_retry_count: crawlConfig?.crawlerSettings?.replay_retry_count, - popup_timeout_ms: crawlConfig?.crawlerSettings?.popup_timeout_ms, - dom_quiet_ms: crawlConfig?.crawlerSettings?.dom_quiet_ms, - dom_settle_timeout_ms: crawlConfig?.crawlerSettings?.dom_settle_timeout_ms, - use_dom_quiescence: crawlConfig?.crawlerSettings?.use_dom_quiescence, - page_load_state: crawlConfig?.crawlerSettings?.page_load_state, - click_non_http_links: crawlConfig?.crawlerSettings?.click_non_http_links, - defer_destructive_actions: crawlConfig?.crawlerSettings?.defer_destructive_actions, - destructive_keywords: crawlConfig?.crawlerSettings?.destructive_keywords, - }; - - return { - base_url: session.appVersion.targetApplication.baseUrl, - session_id: sessionId, - settings, - input_defaults: crawlConfig?.inputDefaults, - }; -} diff --git a/src/types/crawler.ts b/src/types/crawler.ts index eeb1b91..447353b 100644 --- a/src/types/crawler.ts +++ b/src/types/crawler.ts @@ -1,3 +1,7 @@ +// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. +// Proprietary and confidential. Unauthorized use is strictly prohibited. +// See LICENSE file in the project root for full license information. + export type InputDefaultsConfig = { field_patterns: Record; type_fallbacks: Record; @@ -22,10 +26,3 @@ export type CrawlerRunSettings = { defer_destructive_actions?: boolean; destructive_keywords?: string; }; - -export type CrawlerJobPayload = { - base_url: string; - session_id: string; - settings: CrawlerRunSettings; - input_defaults?: InputDefaultsConfig; -}; diff --git a/src/workers/crawler.worker.ts b/src/workers/crawler.worker.ts deleted file mode 100644 index 545a67b..0000000 --- a/src/workers/crawler.worker.ts +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) 2026 CoverIt Labs. All Rights Reserved. -// Proprietary and confidential. Unauthorized use is strictly prohibited. -// See LICENSE file in the project root for full license information. - -import { workerRedis } from "@lib/redis"; -import { cacheDel, cacheKeys, cacheSetString } from "@lib/cache"; -import { logger } from "@services/logger.service"; -import { - getCrawlerJobPayload, - markQueuedSessionRunning, - isSessionAborted, - markSessionCompletedIfRunning, - markSessionFailedIfRunning, - markSessionFinishedAtIfAborted, -} from "@services/crawlSession.service"; -import { Worker } from "bullmq"; -import { spawn } from "node:child_process"; -import path from "node:path"; - -function resolveCrawlerWorkdir(): string { - const configured = process.env.CRAWLER_WORKDIR; - if (configured && configured.trim()) return configured; - return path.resolve(process.cwd(), "../coverit-crawler"); -} - -function resolvePythonCommand(): string { - return process.env.CRAWLER_PYTHON?.trim() || "python"; -} - -function resolveCrawlerModule(): string { - return process.env.CRAWLER_MODULE?.trim() || "src.workers.crawler_worker"; -} - -async function runCrawler(sessionId: string): Promise { - const payload = await getCrawlerJobPayload(sessionId); - const python = resolvePythonCommand(); - const cwd = resolveCrawlerWorkdir(); - const moduleName = resolveCrawlerModule(); - - const args: string[] = ["-m", moduleName, "--payload-stdin"]; - - await new Promise(async (resolve, reject) => { - await markQueuedSessionRunning(sessionId); - - const finalizeAbortedIfNeeded = async (): Promise => { - try { - await markSessionFinishedAtIfAborted(sessionId); - } catch (e) { - logger.error(e); - } - }; - - let child: ReturnType; - try { - child = spawn(python, args, { - cwd, - env: process.env, - stdio: ["pipe", "pipe", "pipe"], - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - try { - const updated = await markSessionFailedIfRunning(sessionId, `Failed to spawn crawler process. ${message}`); - if (!updated) await finalizeAbortedIfNeeded(); - } catch (inner) { - logger.error(inner); - } - return reject(err); - } - - const { stdin, stdout, stderr: childStderr } = child; - if (!stdin || !stdout || !childStderr) { - const message = "Crawler process started without stdio pipes."; - try { - const updated = await markSessionFailedIfRunning(sessionId, message); - if (!updated) await finalizeAbortedIfNeeded(); - } catch (e) { - logger.error(e); - } - return reject(new Error(message)); - } - - const pid = child.pid; - if (pid === undefined) { - const message = `Failed to start crawler process for session ${sessionId}`; - try { - const updated = await markSessionFailedIfRunning(sessionId, message); - if (!updated) await finalizeAbortedIfNeeded(); - } catch (e) { - logger.error(e); - } - return reject(new Error(message)); - } - - const pidKey = cacheKeys.crawlSession.pid(sessionId); - await cacheSetString(pidKey, String(pid), 86400); - - const cleanup = async (): Promise => { - await cacheDel([pidKey]); - }; - - child.once("exit", () => { - void cleanup(); - }); - - child.once("error", () => { - void cleanup(); - }); - - const abortInterval = setInterval(() => { - void (async () => { - try { - const aborted = await isSessionAborted(sessionId); - if (!aborted) return; - - try { - child.kill("SIGTERM"); - } catch { - } - setTimeout(() => { - try { - child.kill("SIGKILL"); - } catch { - } - }, 2000); - } catch (e) { - logger.error(e); - } - })(); - }, 1000); - - stdin.setDefaultEncoding("utf8"); - stdin.end(JSON.stringify(payload)); - - let stderr = ""; - - stdout.setEncoding("utf8"); - childStderr.setEncoding("utf8"); - - stdout.on("data", (chunk: string) => { - const lines = chunk.split(/\r?\n/).filter(Boolean); - for (const line of lines) logger.info(line); - }); - - childStderr.on("data", (chunk: string) => { - stderr += chunk; - if (stderr.length > 64_000) stderr = stderr.slice(stderr.length - 64_000); - const lines = chunk.split(/\r?\n/).filter(Boolean); - for (const line of lines) logger.error(line); - }); - - child.on("error", (err) => { - clearInterval(abortInterval); - void (async () => { - try { - const updated = await markSessionFailedIfRunning(sessionId, `Crawler process error. ${err.message}`); - if (!updated) await finalizeAbortedIfNeeded(); - } catch (inner) { - logger.error(inner); - } - })(); - reject(err); - }); - - child.on("exit", (code) => { - clearInterval(abortInterval); - void (async () => { - if (code === 0) { - try { - const updated = await markSessionCompletedIfRunning(sessionId); - if (!updated) await finalizeAbortedIfNeeded(); - } catch (err) { - logger.error(err); - } - return resolve(); - } - - const message = `Crawler process exited with code ${code}. ${stderr.trim()}`.trim(); - try { - const updated = await markSessionFailedIfRunning(sessionId, message); - if (!updated) await finalizeAbortedIfNeeded(); - } catch (err) { - logger.error(err); - } - reject(new Error(message)); - })(); - }); - }); -} - -new Worker( - "crawler", - async (job) => { - if (job.name !== "crawl") return; - await runCrawler(job.data.sessionId); - }, - { connection: workerRedis }, -); - -logger.info("[Worker] Crawler worker started and listening for jobs..."); \ No newline at end of file diff --git a/src/workers/fake_crawler.py b/src/workers/fake_crawler.py deleted file mode 100644 index 0686c3a..0000000 --- a/src/workers/fake_crawler.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2026 CoverIt Labs. All Rights Reserved. -# Proprietary and confidential. Unauthorized use is strictly prohibited. -# See LICENSE file in the project root for full license information. - -# this is just a mimic and fake crawler worker that simulates a crawl session for testing purposes. It does not perform any real crawling. -import argparse -import sys -import threading -import time - -# Control state -state = {"paused": False, "running": True} - -def command_listener(): - for line in sys.stdin: - cmd = line.strip() - if cmd == "ABORT": - state["running"] = False - break - elif cmd == "PAUSE": - state["paused"] = True - elif cmd == "RESUME": - state["paused"] = False - -# Start background listener -threading.Thread(target=command_listener, daemon=True).start() - -def run_crawler(): - i = 0 - while state["running"]: - if state["paused"]: - time.sleep(1) # Wait and check again - continue - print(f"Crawling... {i+1}/30") - i += 1 - time.sleep(1) - -if __name__ == "__main__": - - parser = argparse.ArgumentParser(description='Fake Crawler Worker') - parser.add_argument('--session', type=str, required=True, help='Crawl session ID') - args = parser.parse_args() - session_id = args.session - print(f"Starting fake crawl session: {session_id}") - run_crawler() - print(f"Finished fake crawl session: {session_id}") \ No newline at end of file From 527edfd38d1adc72bd2ed8b9c036061fc0aa421f Mon Sep 17 00:00:00 2001 From: Youssef Date: Sat, 18 Apr 2026 18:53:34 +0200 Subject: [PATCH 6/6] feat: modify the schema according to contracts and smoke test --- .gitignore | 1 + Dockerfile | 2 + Dockerfile.dev | 1 + package-lock.json | 6 +- package.json | 2 +- .../api_crawler_smoke_test.cpython-313.pyc | Bin 20970 -> 0 bytes .../api_crawler_smoke_test.cpython-314.pyc | Bin 23473 -> 0 bytes scripts/api_crawler_smoke_test.py | 94 ++++++--------- src/mappers/crawlSession.mapper.ts | 45 +++++++- src/models/crawlSession.ts | 109 +++++++++++++----- src/services/crawlSession.service.ts | 16 +-- src/types/crawler.ts | 36 +++--- vendor/coveritlabs-contracts-1.6.1.tgz | Bin 0 -> 25449 bytes 13 files changed, 186 insertions(+), 126 deletions(-) delete mode 100644 scripts/__pycache__/api_crawler_smoke_test.cpython-313.pyc delete mode 100644 scripts/__pycache__/api_crawler_smoke_test.cpython-314.pyc create mode 100644 vendor/coveritlabs-contracts-1.6.1.tgz diff --git a/.gitignore b/.gitignore index 942629c..0f9772e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ pnpm-debug.log* # Dependencies node_modules +**/__pycache__/ .pnp .pnp.js diff --git a/Dockerfile b/Dockerfile index 8cc148a..1551319 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ FROM node:22-alpine AS build WORKDIR /app COPY package.json package-lock.json ./ +COPY vendor ./vendor RUN --mount=type=secret,id=npm_token \ printf "@coveritlabs:registry=https://npm.pkg.github.com\n//npm.pkg.github.com/:_authToken=$(cat /run/secrets/npm_token)\n" > .npmrc \ && npm ci --ignore-scripts \ @@ -25,6 +26,7 @@ WORKDIR /app ENV NODE_ENV=production COPY package.json package-lock.json ./ +COPY vendor ./vendor RUN --mount=type=secret,id=npm_token \ printf "@coveritlabs:registry=https://npm.pkg.github.com\n//npm.pkg.github.com/:_authToken=$(cat /run/secrets/npm_token)\n" > .npmrc \ && npm ci --ignore-scripts --omit=dev \ diff --git a/Dockerfile.dev b/Dockerfile.dev index 81aa40e..d7c3f3f 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -5,6 +5,7 @@ WORKDIR /app # Install dependencies (including devDependencies) COPY package.json package-lock.json ./ +COPY vendor ./vendor RUN --mount=type=secret,id=npm_token \ printf "@coveritlabs:registry=https://npm.pkg.github.com\n//npm.pkg.github.com/:_authToken=$(cat /run/secrets/npm_token)\n" > .npmrc \ && npm ci --ignore-scripts \ diff --git a/package-lock.json b/package-lock.json index c99662c..32e2578 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@asteasolutions/zod-to-openapi": "^8.4.3", "@bufbuild/protobuf": "^2.11.0", - "@coveritlabs/contracts": "^1.6.1", + "@coveritlabs/contracts": "file:vendor/coveritlabs-contracts-1.6.1.tgz", "@prisma/adapter-pg": "^7.4.2", "@prisma/client": "^7.4.2", "argon2": "^0.44.0", @@ -948,8 +948,8 @@ }, "node_modules/@coveritlabs/contracts": { "version": "1.6.1", - "resolved": "https://npm.pkg.github.com/download/@coveritlabs/contracts/1.6.1/2a65eea060d869746a3d556a25e6f43262bc2cab", - "integrity": "sha512-1YP5O2ghicIsgeWMS7QGFjfc+aLxQWByZ7Nc0dQDnZbNb/GEFJQlnmUFXmWw7xM2PFArUFBUBzwxc2PIF1v0zA==", + "resolved": "file:vendor/coveritlabs-contracts-1.6.1.tgz", + "integrity": "sha512-eSTN3c+rHiKoMWXJFm00RUtouILK0YIUcyXNDoX2YjzOvJluwdKL4aJ5X0o1WBySP0IcNglH3WmCMXwakSR4zQ==", "dependencies": { "@bufbuild/protobuf": "^2.5.0" } diff --git a/package.json b/package.json index 048ac86..b703d23 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "dependencies": { "@asteasolutions/zod-to-openapi": "^8.4.3", "@bufbuild/protobuf": "^2.11.0", - "@coveritlabs/contracts": "^1.6.1", + "@coveritlabs/contracts": "file:vendor/coveritlabs-contracts-1.6.1.tgz", "@prisma/adapter-pg": "^7.4.2", "@prisma/client": "^7.4.2", "argon2": "^0.44.0", diff --git a/scripts/__pycache__/api_crawler_smoke_test.cpython-313.pyc b/scripts/__pycache__/api_crawler_smoke_test.cpython-313.pyc deleted file mode 100644 index d7c906321065baaa5add23543e2f9b9bb2e6dce4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20970 zcmeHvYgk)Jme|$%>VX78yk7!@L70~TyKVdwARYz-*44GK!PW@rf@KSdzE`-7x_g$M zO?Gg55}3_Qn%?P+>Fj)I=0`GY{)El$W|-;DhRy!itGV7mdxx*n-^SU*`|S^U+M7&z z@*}6}>PiCBZF~CLFZuGNY^v&3oqC-*bKhbmTuLp7_$p@!AsP?rix94d$O7Y(f8qLDRTG_j_O zIc$!dpj4A)N)3OSNpq)`(v}jp)vP6znhUA9kgCI}d8t$@q*@`B#Hsm|9+x(Bswo3b zu~7>6XQWImDk=x^%$Z|D zpzdzMs+MxJkZfrR8kMiZaAlB!FlL36BRHiTe<`VY{8ceoNi}qh z6NKMP5S7GaRShv&Q9~S0P3duk6k!xIifUrgt|R<NzSyGB6u#O;63VVwyOssv@Qp&ZZ}*%!pGhs$7u;QQaE~ z#6)s*E*6SL{9#e!ou3OcqB`QAWkg+&nfA|zW1^O2V)JYy0OXi~_1@9Ejd!*ifw5KNo7{0stA~S|RQWu>RM=4C~`& zqjwlzjNxJ}bASsURLSN+VH5mus0WM0E@@cSzpYKOIfQ>vaZFS!09GTSYJuTIb&Q>7ys4}})Oq)gvVPrlQd>Ylm-yZSEp?fN6ti+4;6} zrT0PWVoy?McG!~~*H>|k{R^6fCo zoU)2=5c2wDjgals97M1x6p4v?_q_mv7iu=avsF++CV2_RB*rlmVI7cHf%4tp9fso7 z+K0|}oez(`3#bHS?=HY&genqlt60x8x8`2>+hxfedZ3yko^`2$82Dvw z10s9dh(>SwoI`~Fje=rg)l{$g7m9k-lS)Vcrlg4hCy9vasc3LP)Wt%xOmrUj<}4Gt z6%C52c{a=*1xU6DdClOZ2^0|~nsiK{SHMRCKcWFKs%%q4w6tmpy7*k|sbn%cjHgoR zT&Pz`YbiAvI;~9eSRhMELo3Zh2g=o=+zBms*m=@KE2j{I2le=X8!s6Dng{WO15Rd1 zlT03YN8} z4aIQ@SncYdKjs&8w-|qrVL7%1dietdUe+SR)CL); z92fJ)<~hvEL=D5TQC8HW#@#rooSqX+p7{u3MpB9z7b9uUF#dL$aL?IR=+go2#CD=I zw5~)I^a?f1&0$F+DnKus^Dkg#int+2_Oz44kHL2<7MsI7mmLKNgdT1dR>vZdG+GvW zKOu8@t7A{4v}hOm_p}6QO6Wl7WSH-cKh)^slER?G>q>SLWLS=Y2q zulMt{D9w@}?T-z)%Psd?b`17}!M>KiS@h@Se_H-$mAt{u8^#xV_SAaKkYYC{fBEM9 zn>#tRiJaQC9v+3KHoZw};XBT6JO9bi9jhZ@b*x<#tj$Y3$^4>sPJjFKzpH&@Kehqh ziLFjPzyE(|2y@p?&MEl+tEXDRWL-XU|ICiDI$^B-j8JLD6v@V;KXm-ivE*N#zCZoo z)^hZIbam#D`N&51lIr88)<;%o_>r|?!@H#0HJ7f^YsWq~@&1XmhV|<8zV(wE$2O=9 z_h#PaE1OQftXD9*w>x=q^jEvaJgFgj!e~dL7zP8Sed|}&2R2+Afem`IdegPp!IzyE z%onzTJbCd~NKy?cKIeJ>?LTg-=&2`ul5@VNQ1zin5AhER4LuIshjtZ;JFs70+1H@@ zNaJd7H>>`Ab2WVYiMgP!R`n;f8i+qZ44FLvURq?qR1%2MW0CcbfgL_ir#eh1(P*a# zB911$Opm2hr&JNu^VA0q?4b_2@pej$g@&3|A11ESLgNsTMQJG5`JPfnG*A|4xCb1q z`3mA7U6%(b#0Q)QDX#|{Fh7w0pbBUY6JKd@0GbFu93q%sWTYSr$cQE*{3t!tb%?kE z#%{oj*FYmCZ=f|G5{+tL`_Hg?txF56K0TI0M|-&NL#$q#)fb7?%PndTVfDDJELP7R zgLdPp#@0rs60Fir)qVZ<&u_7#+Q5_zYhob7^`)6BvF z4Z)s6-g)F*K%NVE-N-`=jqODq(z4k;W;SN0Nk zoDRGcKi$Vok%rCEkcn#MN-#Iw-Ov2i?`;CM- zCYx4x5W76!GEfKxHs#6@QPJvD!ift$*$}m1>4ZbcUI761DtH_)Tp$sNI`qEV(NM&P z_7{_ojCNa`13WBhmrvh6y}}8`vef{{skPn@hTb3APzjFCgt3!XcS;-O?7pj^BxdSA zgUGMN)HA0%3K(%(^M&eCF3qk$Zv_ZaPS1ogL=YJ7M4FvZAShtrr?=D07t{|-NqKm^ zP+A2TR*l4S1|xU<;ZV>HnoyTLuIg%Sb!zb<7d3N!7ABnpgmR!G!WNDT++yG?OdDZh z;b;J_Y2YbHXTrcSZr^<9e4r=Jgd%JJ0N@mm;1H3(&Kop_;rqiors{;LdaYJ49a(fI zjk(Kbmd~utKYZii8)$*tSnpV`wJFBhfW%%A;X6n@F^?WR<0E?=FSe#M7k*27lM}|g5ua3mYRHQ*vI7NleX%Gz| zE);=1QzXD(nih+EJs#K9VISpnd1=bmea$yQU+Q*yL=_w_iz?u}_;?e`dr^zm8z+)X zB*?uMirvB_4`-feN#K)C>GTtNfAGw+$QOySv%qoUjCAzuOYPhi0XBx<{}{aVqA7l? z%Uh{?FMmf@k%gN0Ss!;9};T+dtYTtBiF-zjew%G;AgWxs9w zR6`UThwWHy-tyO126?jLF=<)RJTT+q*0n_&ZvO2b`QQ(~}=gNG8yN#*R))-G~xs1ZeQAvI~?rKZ#C) za17RGB~YgW=#o-WYOLz3C|!#NXist2nw49OsK$@h)isC`vk|Jp|ttkG{>6OdhI-Sh7Ee-6c2vZSITX9L! znx8BvOcoYF`lm)MJ`^w$M$>Zr{rU$^d=>DUUxyRu`)}}M*{-3GFFCu(Y+mJy+=8Kx zSNBPz2VF@!bS#0Hc%%@kF-QhrhjyGYTa9F~T}nl%)9G2v9uY@6U72D09w32(u7;p= zGBZSMEKQ&VzzkJiMp!ar@CXnonGpaU!@)KPB6E}IL8Cn2U@xXedojw;0lVgm(1IJ; zuUfAO@9{F&Q$H6QIcaQYvU&pd!C!HN2YaH;sy&y=OV5`sE8R-vr_;%awD`+{uu%mv zM7rPliL~I$f-Izpo`EzRT4ym;@(iRgBlf6Ls!Yz0n6lE#v(j@WfPN&0?x)l6=B)Ip ztaM9e-#n4tzaMtUjILy96|JUqw4OH5M#|ozLpR^2(keCxZj(%#{x^WlqMIb_GRLqP z(M{E+`<6RVEkjZcI+v=Wj+|1>0~?IwMXVE`97v9~PB`SaTsGYvS~XTry_}y3sd{;S z;GX?8>7ISUDd$s8+PZ(6@Z$KO8lbO@Ey`!Gkdy8eY}O}`f{PuDO}B4s`E(w2vmE zS~CSAnAXr`6dce31zYPy6PrO7r$);MeEnwK7DEnWKs;~b-FzQQCrk4 zm6~3PmLX(XJQVH|aT>NsU99G1a`7PbhHI=b$K zbTC+1mq0J3>&BFq;6xp5UpZx@E=>P?P`1`*4H0neGERiaOxrEeqxA8UP-W!Qa;? z-%FG`O#AUuQ*sS5uCwY7yj1;}eg!E;u9cbw>YHgHBS$7WJpPTKheE9?&4uz!WUaP4XBCDQm(GT%uzPRP?r(2ergsc?(> zpuVzC6Og#B#FM18I!ilY#YQ^xG9DvGv+x(CkJ2S=L>6yqdZ~UkQFAZNJ#dfU{s`6{ za*^f}E3--&Y6;lj4XYW^BLI5_#e`3 z`!$-G$}hloh6nwYOfxgpGv5S5N4aVOZWF`6M++v9h?VB9K*npNkUCBmqLRTDIPW}> zibGlI6}r&#CE6@5V*|L8&xiBF=J_S$Lm6F2oqASpsY0f>1+IdAa^zJe70h&K6r-ZeJp>vmCYAKWv?%aKtW+%&@_vOjz)%Eh`m+S=OCbA;Br$HW4; zjMq(hePb?)x;pCVja$87J~-y*xYwd=@K}3C^@Un7Zy)H0>ykU33oY3x^q9ep3V_L= zbsv!X65N!=4bWCIe)7=#1RWm7O&6GZ{#khM(GrNxX1YD*8NKN4@%m8JPe=e1%jDYR ztX?_~OeXg-FTr_Ts`B}I~gVz^x&CftR4=-R_8IIi&3lCHd z7atTf8rQV7w6z=)E9C64(cxj=;E31lx#EHgJ$KLONG~NS+QssH5Z=K{?oryC22h=7 zYZL2a5J35m+v6J>>=~lRd-N})V+zqBG zG#eK4=c3{8AQ-W-a9v4(ohV#ZidK~9m2A@Gv{Im`I}AG(3f!5W+n*)^@njp-g=|*hN4%I;?8xK>nPu1`(HNBV2X6VN!i>E^x-bP>4JWS1CXITG zWpIO(WC)9RN8Hrv2W#C{E;JLFpA*$%qm&n(ORPbbNY15J627U0hXPHI3f(eEz)~$> z@qvrRNo>d4*>+0Q;PnGG{(fZRuk(kI8SVlU%t;lz69PM3FwBK?uqKWhr7nb{U~3C2rN@~1&TcC;#w$`8d5kGQJjPU=e@!Kn5D&CcLmw5imAynww{V6L3ruh(U&J!ls*c zDGm&Vi8Td*Rm4JffvTH;_sxbPC>CU+b4|?bTx`J}@UyX|Ko|`AaTL~22|R9u8*Nx8 z@Zb%VWqha#L`1giM7R}4!k2TBqJGwY50MMJk|QD73vUT-iPlgg5S|YrS|9VrpmC_c z%G`SHcj98Xv zfaDm{6NOg{q9w-qBOEjyjYtV5Y`A*~y6c}|L_My73A$pUVH$eH-9m8?o4u$aJv>LG zBq{M<;G6wx%B}ORyp2-A3Lm@Zxp7U_&SgssIB7 zWZ#e4AQJ;o&CM%dSJi`FoTiZ;;_O^36b{=Z>o&W88tBG8hh?!nMFjT0TKtY(64Lf? z=nfNJz)gWf=jMUFXIkuTh_Q@4b}Q8|R3YU;1X^I{BN2eJH^{GGaqk$m%RaXN&sie& z=2<((23n*xxE6S-G7t7ixPFi_c7G%Yb^BxXP>i#C+`WSo(qnOQugmL_6*&7G7@)@N zu_*2c2NVFqWbcMRz}fvAP6YL!g^d8l<_EyrkBSHb%+?NS2E&5rlkm}+GWNqLz|(4n zS9^H)xBP%++~eRmWQ1{WhVbMdFJ2|J1DY);xJVO0Vl_ifXX7gS8GGDu7}NnmI~#8} z6bck7s1IK9WF+)Cbc8AO2@^~dKAlzgP4U4eo$C4dP!L}DfZr*qZZY@Zc`ZDuNg3?e zQ9bB(9E?m^WM2vU8G=PSAZ+K0%}ep)8a9rA)IoU0&B7gh%IKU%u{V*2-Y3E{X=KZa z4R_c@l%PgLCaQprATP`?bCNOJNy*9>=_*)?OKOuTCH{T*QFy-mm8D_8+QjOc55%+t zB_|*)09hvkZ<8QWqE*a6Iw`Uo#hR*U!k;P97tPXA@X6oM14+??2124G{Uy~Y8ekmQ zqz)Lf1;sNH$cY9?F~ug*h!wCS$Xh^IFi7ef&bovWRI}WSsK?3`?le3r1=UP6N!pkX zYj2`eQdN919?s}FzCn3*dULJgVkS}Z$Oh*^p9~*O5H4lyN z8XxAoo0GJbzvq2;{dcZ^Z*r%iIZ@HP9$07jigv!dW8(~;dtpiQIKPN5ZrN!3Bkzx{ z|J&<-IJtB7QsV5T?Wyfo`Lmb#GakN;67s!^!@H&B51qf`8t#et98bqu!^^z-HZu^y-;}`RDOKpRUz-x;>F#JI$Gm4HYz3bd81;sysvM2>zgH5Z zZMUHEz1P-a-+OaCBov%nGVGQ**68(%TgpUf&wb01c15>qDOs)me&>#*Az^7)@BiV@ z4~8~Lp~d(WD~aT`+BGx3=;EJdLEovJ+Vf8?<}sYe_OkCZ=2((Dc%#{tAc`+ z;Z01s(v2fqRf&?z{Pl^1dGZl?<8fZ;YUf&|kk__&5xudoq5)i4zrL;EXJ1X6V-uxZ zY8Z02){kuVCrWy^FDJ~SkI1p7xdc?%Zk#ZhX6O?aBu8Z|-ceJIDJH$NRSK3dcth1*81bjf)pQHWuLSbZ%VT805>&3Fh-Wc|P5p^($MAiGoW&#|gvuBlWe%1(mC@ zwfIKkMmbs%bsnLB0wT1QCrS9-hr*<}G+9`e$m#x6ugTZGH3Y5p?3oCQbvd-+S@zxc zt$Kx=n#I14jd{4$^INLTdA`gon2~XGy48)l+uCPG-L-LhW0H4W;jiA{uiWHs&hXA# zLP-b^0qw6|0T!_N22V}$H^aPVHeroO%_pq|?>JYj?^J!eYRx9(9myUtyx`=^x&-rS zo;;my{Fxz_)Feq$(qLP;zB0_)k8ezGjBj4wHf%v-Q#={?Ort8*LMNV@h{BThY<%U( zO@~l&UMRS*Yb$zB%~zb*R0+jr1l!r&#umPHl)p?1jaPOWClig6{Ee?YYV-@0(VsT@ z*DmwS^bQkFFyS30^6=F}qd!p@mCH~qfe-u{b{2Q$^?Qtf!m3| z?VUgvmGLDiXLoDs`G)@OBBA#3PVLo1?Ny=n8h;~@s15Aeiusab8z%%?*N*K%!gfKh zb#F~4Y{1bf-V5@z-CMmv<)Bb{@v*a+Z-K5}6`bQc&YKD6O~Ls!K6pFfyuIsaNal9# zDT%z062@L<{#2taH+-6BacCEZpVknz`tQg1rh)ASpa~J({bu=gccO4q>Uf4|`nIm_TPN1C=T=Y7LEZV<#i6GLBGUr6uK5i@nh^{ZUOKfgBktw%7de;mPdyA-GZt$&D!gm zO^Jg3`-UaO(kscrig#w$tUCn_iGqgpVxgcN*mY6MdR!>#S~7i7Sd%QNO_tR>Ji9iv zQ`(#;ZC(!xrKgkj`VVr}+jpwk6IJbeN3T%jPF6S`-dI=el(#0zTQ|%?`T69&378N{ zPCZ)@VA^I_B4J)lwv`yXXWKR9eaG=m<8L>vM%Hif6|Zb564vfVCV0DDw$%HHv4}78 z2u4q`*s)F~irbb3;c3KUNkpxjeMDA2Z6=HsdB8t2)bF*y8}zr(6D+>0TQK+VWX~r$ z<2UcUb#H|fbS10fg02oS_ms+<8;Z|VYKviyQ0omq1*9nhMGLwbTm>VTr+ISvaT=-Q zBZK|_1*GD7-e`c`nwd}*mgrIBKEv#4#KD_nrt+h9V%1)v1_)cNhX5Nx< zw+w{O2PfY@$v2%9YR?H}=Xc6_;c*LD3{qtE=(<5DZO3x?!dBT<&Njtgp5VtO`KlX& z^(Jq+`LVH-uWH?B6{=hq%FXv(;oVpHtH9E3^Vje2Rbjz8%bRB5725K>`}bCjI~GU6 z;#lwf;lK|DHYRtPx)V*^xai=vW4kb|MZhbW?AZsEPSCZZM5F1KPvL=tsrWNO1Jd4{ zw^IFJdP$Xp-sTI&@-^Lh$M2ap>Nlw$UHgM;e>fqu4{Q(e*CuzaO(m{P3D<(ca4=1{ z>#3E^Bx zxS4!ZrUm~mh>~!-;V&w*;QvjhA>63?n{zIExK8!A4h8b-G>{~cKKQX4*bVuf;Ikox zsGf$Op}>z&oORNM5?uqqL#G9f{S0F4-=a?w>`>t6C|dC8iu5YdoJx{^ph5ST$e3G7|pv^7*=?ZBZi+8M(C@p}zY2>|* zyhY^QM;=n+(V);a)aMfo)B;>}&ARWwE6uziW??GoXM_0agPos?;RlQONj19%h2c~` z1}BH?4+#8RMA{vYXJ(_p`7m>i?S!h~KfxV?4Lm$dR49H<7=KRae@+;11mgKWCknv- zPeijoH1BDZic?G69s$8Bg@ZNfDf-$o=oR@ZJ$nQMYYrT&cRod5d&N3M(Mrc20l`{6 z4%YLZqOU!hT47s-G7zlwJVn8tnQ$rG3V!Ae|HjvUK@=*66f4ji3Q<`U!Yi!&-_Mca z+XFm(jlX___f7HFrxJ7^F&va~_`3^I1hCMn98^MZQF#dkqe_qT|KKVX78ydM$Nqr^RM%|sz zJ(&r%JAovH70xmel%#;5qy>a8JFKN-5SHVxj#5CF#Nlj831Jlu=TK@&L}}nJYogpDp|p^q zLn%ISSPyYJi-^jGxSV8ME~STGxj3(ZGC(*Fhx4d>2p8aRKHyq7A)<;V#8mNwgesYk zQl%3zs%%0|l}{)rqlF9?P$qz{!0?6CJ_uLha1m7n;r%#Ve5r)0Zaqg3ZXH3C5+%f? z(hA~IaRt$mBxMUw61gc6C8{J$#HBJh;f5Y4C5#w$;>|R~rE*BCL1|&*B@>iW0e?mz z{gT+$&6kzwQwNTU`QOR%L;^gjF>LHGdPQF1xTK8mip({?L!d)uDJSXh&v4RiuPe-v z!@;mO;CK2s8664wJT8QB5^F*ZVF~>G9#{o9!cI(xTfYIVg}74UtO(UH4k^5H~p zI&6_t67NfYhw$X2e&@7@Q@A~o&WJC}$r(>L!ua18agtD&F^f3)WPq7=hC>K}eOzmI z+qf%m-NSgp!nfWT^p1yIj5in#jYHVsVw^X89>x)x4qWp%!k$pLAvnWn91f@79|$|c z9*2X;fzmYa7peenmiPl979INxd2fZb(C6+5{@fqIk-rBZ!j&W|3S$yrGF*ipvqZXS zwoE7tN)Q!5CZQ98V!PN=gotGq+hj?AjE}%32a<6-VeR5df(CYZUo>QvaN@4$}^yl4JP^V}g;+PYEz9-{}~KdJvyQcVnz!ePI@^I z@R>ynj2VLOX^cAVChVd9O7)del=~S0-rHSU=ySgce}3G8BYBilRY5*P4BS`$3IcoW zt4YWSeFe0Cc7NFvQ1}if^kq`NlrHD^o2ZTDU%1;n0(|FpaNvRzzq)r@oCPG zz{p?fcyBjvq0jwX{P{637yNpXj}Xl;AKk~K+2fdpK6%kGeuj)o6*pdvshBlSUxBH}Wlk`Tl@V*miNYzJd7br8!btv6jBEax+22ZMye zcIiQ4-T<;P7D&VNvqE4u)>y2(XMWdw@8Ay(VkX%VOJoI`I{iZPottyIKW?fGj|q{i z0Mc%2wQe%-eszZGy&^SXCxgZO)HY|6w%VKR*AsO#aNn@@>fpWy?K38GuvFNV))@fq|P7E|Ia^otUIj?zZNd_CKxAL>bD zTHqd?1AmsFh@cD%aIr-cl-nhqgGd6U``{^7kVKGRyIDR|Diul=+vSwB8mhF55yv9u zhQVA)P%^t%Lo}guIZ99WlPx6?F*x>Y2(Th@+9_}P+-q7+AyToqKnAWm`9 zuH_>Ew!@8uDN-=OkBb~2lOqyGs`MafCv8ZF@H2xlO2}<^62h)bQ3)6i zdIDJh*x8i=&CiG^q8V_Bh$-kjifBQ-3C;)_!$cg!k^%j`0z$Nu@QRHf?2Y&z1rYp% z0rXCq9>|c`;cOY>R7e7G7DOcmRkX@hD#WJ0Da=g>3Sho6b)bdHppt$nE$D5u2&%`8 z0u~g)4eH{JT%*C zQ$cO=-D(pwyg6%#lXtj~CXZ%2gF&Cy+`rxWkMM79vaWW6X1Q{ivyr3JApULE zV9c50fYVN){1S@mK*&k}D^zhPh)o_&1XIwUa|UZmB)Y4stE#F@yyNw$$}trV2eDFR zE&>D=&=C=mg$cE8w);~uo7GosNyIw&Z10wwAT=K=!1_sR*9~3ux~_WZMoed!lP31H z%qbSM>!k4$g(k7TZh3UMm#su`x^>d@NR>U`aI0ZMWm;F6mU34M{=W2YOaHE%RheR{ zbF*DrQl)G_w3(GVKXGefBdc;ft8%G}&8pnUIu_45wn``T`R|y&WB%U34Sm(RzG`VK zrmvstO5_&2bNoBUKd6kG4y}N9WUYnG?frv{(6&KF_#ewU_T^(aq0!H`-)i4bSFEcm z9upFoO_ZoT_>-!iR4rG{Ip-&DP2QQB58MhYUWsd~S32h;pVV36dO#qquUVnz6r0-O z#nGih?;ZKkk)@jDishc=)|EpmBP-U`oYkYNX11g|rnNq3VaeeyH`O_ORi<^d3E76w zD9G(twlDXsbgZ~mMpr9VJ64<7l2bA5OKWbHJpCmyNd`n;godFDetx*7tDX3;tb^#P zk$qIAf*&8%sJo6SKWdhu@G%MU+l{><$uCOUdSsGc=3Bu3mCOV&zp54?|3F?(yX05x zG6-)oV30|G)!kJH>~WnVWrc#Nl2j$2s8X?MHM!kDu}IMb5StbmES$C`B)m0I2M9ga zeCP$B5Hd-S`Jlq)1M-pifWF9lK;NHM^536Delhav%C&QU{q@gq{C4#ZOpcchFg<<^ z?U*IfZP-oK@21}EpnfQkMPw(QHC3j>N0VwPD0!X*Z9~@m4mZ|BNuHs6rFQ9VCP7O1 z?qU{DQc6b2g))OOC;=I_JDgpXRILgM{_R1vDl$}Ss!UsUDwqn`&{E6}2P(vkRfwFH z?`9UHR4b?-P_6Pmt?i&*$)GH4T2M+TaTg;16r~EQF%?#tDq2RVGmY@pw&pMkU~RD7fS6$9-0le4S65KC>dgU;D=eu^rM8+$QwZ3Ao9*2ZwPrvb}=^O zokbpnyb`g(tl)Ln0Oh zMy#kdx9gSVo2z+j!>I@Izt{Y&=I^vvy^Mi+5tDmUFDvIk^TqV1QaWZy!sb>!PqT<& z{*Y)t@j>?o-K*W(HOsPhE)dtYu3es!JWHK4uZI8n*y=&Hq$j5BeL%D1**{68^Z~Jc zTGLre{JhlCnJ@cLt%C4}`RdLC$`31~D11PI{92>6Qu2|qqrCfoHr9 zYq{i?}^ON>~JuhNXAkGQc!}ki)=}G zF2ta38afje+9%>f7PA;O0r09eC-?EI+hXQ2X)!n7U-q^-x{0)cxMTj|NsGv8tAb>K0bol3u_Onv-g1BQD~_*DNv? zoA|-1fdZb}CukgpkWk|nnHtctQp4MznBj95)*#`N#u31-kppyaqC!ya%Fy(lm(;i~ULkVr}yziE8Zep<+0loX$qMudQv(ox>_X#Y9 zR)X+YLVDr_BmrV}?9z5*rY1#<_kMDfQwG$Ec=&ZE3BS;U*PYM^dzNT$KzgVgJj8}_ zwAKk8S`mlO-PZeE|MCQmNw)>#1D+KAa9fa`Lv7)=O3-2fSZ+d#Gq%3aGDF%52HOSj zL(A4>AP;sfT8sHrGBn8obE;h8d+L3XABpx!IQ8hzh|Sv7-_vjH=HzEbt)uW&P@_Xb{X@N+ zvTOK^ZO}@iw4RRsL6p|fIZWXQTgT`KL`a6LW81`jia;+YPg0$Yx|Ly2)5Uy8B7^f; zeHe!Hlqk`Q@;)QLyAOQ`pC32lUr+X5N-KQ`eU;IFqRMT(m+rBaB!a@==a(M~L|qW9yvx`Y zhAl!5wq-MLRtRwEFKio%?UJ4JM5V&kA*UD}8WInk!!;D(`T-O%(8hL*YRgf9E7nZ=HKr{6ujoU`1rT)h6+QYGjH z3Y{?$!?_#@oXe5Ixg6Pq)FOiobtHRH5@RiyIuphpvZ1k~(PC3%mXYC$EQeV~8@1Uvv-d-3eiZN?v-|+B2v`R2lx`ypm zs-I4dtR$R+K5w(OnP6oHjVDgw4Y|Si2=fYV5LHHl2g&KNgoY8Pn1%&ium1`sx#pP( zAsWJdy)enkbzmFF0f#@XT!FJLVOZK_z6&szDZUK>7*R^?e92tN!oCf%WSuODk!6^d zzd7^9%r{?s^VLm_Zb5XXXHJw*YZgS`KE81FTgMZ*hPl2i385)~v7pRfcqKt*FH|JR z+=a6V(y-u6kU0y7w$yUG#;GOLn)w5_4%{*0qXFOe8tm+T>ot}v*;M7TMJHB0t7B|| zHKyuer9EjT6b1-VCYTAUFtX3Kwk=4&iTR~5WL(3rc%EHz7r;XKxa>a=uSi5$pJD$k z7ysoEg5;D#uo$85LqCCOEg*_EOGJWh3TDQOVPht#6G7Lc;bFxn6)#TfPaBd6J6r{v zbrZA+Iwq1DIe-IbA<472T!6g-f&w1o>4N-81qsYuL0{TVSBne6~ z+NAmu%ip;IBu-H%V>|aL6w;(9B+I}ki_-o{Eq*#i@{C$^RCWqeMMiuMrBB6^<0TB~@uxQB`)eMFA3gzp#!f zP{c^sH3A>I{{m|Zbi@R^^tM>qPNJ$)_-2n+2$0kPd-jBisu9+XZ@{Pw<^=WQNRVP@ z*N-DPgPl-9s$PDq>1{%U00>{oEMQ92w%!0u#ZP$60tj``uHQc4-2=7is9LJto^267 zg)&(JLT*?N1U-rj0_>o@ie1{9YtNZ@(5eR;7Hn#e9ywG)>(2mZh=JT`oGtbo7~zd} z1z;`}+BYTI1LlXI?0UQ6X_$|v`Vjb`NuYPo0Q16hpEcVJm{(e0XW+114Ci={Jkfv9 zGrW5)=N$!%TJ8GhV?-0wODT*BGV!4cC?hSQT4<3)`~)tAbm5a$l7-lnXF4pYQMGG& zr$F;y5nUwoadIgo-Ai3`;aBFYZaRmu?g3+~fYA%luO~&nV(Kt0g(Dy(w9EtZN!VkM z(Ir%GHFVlG$`;c_5Z@;ptVoW}Qo8hoY1%Jf7%Zd9o=UURbm>>7*#OXNkS=>63Y?+K zU<3`N=d{7{ZS5_mi-0n6YM6#G@AE)+?Zy z4xK5=;1_h#VTY?A(g!w;z%2uy_vpq+SiRUbn^_RrFeRcJceS@uJOMSs3cq`23E$a6 z3E$ndgzu$G$Y|5E$74fs4DhqF^t|Z*j{$=E=Qsx5#bW^Eb}6xE&IhskT;BdR=FGpr zqvM1yl%A3+CC>!DeNw(o5Wyqk&nqV-;htH}QCcfl4(=i>1CG#I#QjgKOuS7W0qy%Q zXf0S187mXCZb$y3v~Jh@-=~k_{JI_acb5w(nVBjb*0*2|yE}ABRtnHh%-G+d8@F3Q z6R;Ebl+w~N=?9oEu>Cy&dma<8)D8ROH9`-VS&J*Yz%%_gg`KW{| zO$dr=Oa9{5gpbqtwr5z#IA1=_*ZzD+H*n#e^uh_Mm^w-4Kh4sCeg9Lzyi<8ra*&-$ zhSPVG(LO4ge|W(cLgpWG24Rn-!`AQU>=>~+;Cx856rIh66C!Z5qO9Wlg}2xSVX-PVyVs^3QU4-at#LL%Kk^;&61hs}m_N7Z2`1KYK51v*&& z1l04W0|)(m;VCYEN8zxAH3F4JWsMDu4TrcgA=x%OIOymfqOFv@1GbH=UBg4&Bb=y- zE8PY`_n)y2kJ2ds$&toJZodEmgdea{4qJcMz^Ki!6IR^Zm`MmLY`b+0Kx_LQeb$aa zx{ux^HAxcS0MFjl0;J-LnHl&z$5s48QeCQhARzl5r7F? z^}t2=ES|cZaM*5?@o~AqfX~+tS1B0S4IhEq3a}T>=}{!j-+&Y1ih-f7c^I45buALy zo`#N)di`#=jS*E?k{uv{E@e7_3-Fmxxbk}~x*zczpA09PT-QtyzvpHUaDw|5=+Z*K zAM&)BI3+M$AMQnb)(M|MjaJi`Crs#!AUZr{f=lm^$?2MMUiNuRHT?EFoTxHG2?oB2 zHpYYN8Nqiiq8f`6?uJ-G-Yfn{kdxYmM`(@=CPxb1>4y!tqku7|;bHiEa1IOSh?4lS zNK4}}PKHMZTo`ns3xf)$58d^c;j)t1u6f}q2;BXESh%n<6jk#T`vPz}i<9G9h9OSv zgbOaA5RL8}M$LmL%@m42oW~91xEu+aG5}1|-VmI0xB_ic<6@GiNh2W-6IEK0eG8?! zJ^Z5qa9swr7Ll#SH%aVuM+>?Dqp-)6gic;{fiiPlIKjqqNV05Y60F}HRi!GhL=Beo z=oBvIgg6E27htF8zGNLdkSXK#PP)1>zB;}~JJR=ik`>D!6I7bFTSXRVEWDEtg62V` z$xy0c$=~Ukk`1%-IfRBK!EnQp;gBV&v*0Hd>O;^gxOnBX^Crpy_tXo8fySRMIn!}Y z7=q%%jT{Mp-EdiHDlM$iaNSlL-N%WVIlb5K@;9}SNY5CL$v$kpv}!GRe>x|ICmE4B`G zbPifUy2051y1$D-OWh1I-Z>J+tJ?=8NlR2d3&^hSLF>o}r%m3J739Q@qMwk4M;bX@ z@-d4MPdE%*9pZ}Mp$ypX2j+#-Z?3TSy2s&!55m;Ao%G$-9+0%4MrdlZi|)5uIYVYH zI8=c2UWnVbGg~TyW59YIY08o4_hBf(72i7en9vsuUmS3IeDDrM9pjmXM>*;u{!rvH z8daA)buM(&$XAEO3d#bvtLlOg2Bc8PQwJ5pF(iDb-o%GNs(He7ZV*$@Cg7wx==N!^ zABEgZAXw*_4u)q;E+-SNbNS!~8xC^C+i}l4alza{ouW=2kU&@8I5j3F+=iRo@j-_2!t{ob$_lmQl-T+X(iL-4LiG<(NTu|(yT5RCXR zC$WvvoF0d_SCWO}sHBwd0ZEHw6Q_i?8?03z(^3cxKmP8VjJ{t#dh@beKGd_&<3 z;O)HP;gq;AkGmtxsU`uv&=d}9G8@9oARfK}UUYvq9JuB6h06oHW!g?1w;BZl5B7Kr`fknylOQ0yw^Ow8C(_U(NF}s z`bvYz3L(a03Qr{~h9dY>2tW-?#P0_ zTvLp|p)*`Qyo6zL`rS~rGi>sPLng}F-9LhiM;zVVL3arDk|_v>%EP8`03#BDZiHqs z*S(;GOwJIF1ohm&ybctLzzXvYQusX}DNUf^Jq)NPo*tIuK~hWt+$AQsl8>7|+XMRG>V#rU5w{T?7amRq1?lMzy%GZf~ODB9z8fJqF`RpE?KD;E8@22C=I z&&qOS$nOk>rUGG3jpC6ULh?F0-D`YIs1@i4=lZ^qOm;uLvN?l==|EI zTE}Krvo$?2b?elhRwY4yEm4?KYV@J8_R2*Q*9PkEsZXpUK7WQyKd>`%ES$wx0Fx;9P`Xs=z6Pt@yOE1-4nRL z-Usrvn-4$%2Gzrvnm;>Dq4%Tky&ELX48toE`+-4D*jwZn1J z_Bfk>0;`s?V%ZI|{RrvnOEasAwY&!^mL6x1UWywg-Z(ScGiOa0iryJm7+9n>a;w&J ztCr3!GqK#Jjoc&axkpxBkL6lt2R~64!0jBitY%oowRYBiIiBx&V;FGkP3Vi>sb8oE9CNDH zbE=l;SWfNi=}**o7_F9-v6X(dos0#+B-YT3LqtbHu5Iv1Cof0S3g z7+#95)UK4WFqHOFvAhxBUwvtUgvYn?6WZd0*02y*npv^1S)GrSvRwHa1As!;mWI&j z=e={@1!~@L%dtquvi8mPe4@_5D4bf8tVY-pYfOty&}X1NoF$KE;IJDRiuNT)O+sZ@xUewDnhvi_uAE!F@IbW&bzWvk*Y9PL zVmaUftyUV|9=U5^%Uf5gVnwH7c`qeM{o7r4rEJ-eRY|O{J!UwOAWMH-#af2hv!k(E z`$p~fdhIxS>E(E>Ggcm0uXQe+Wj&J{9^bmhx8d>Mdu8zzh+HoZr1Oo$YUz#I3+uHP z*ozbKT1TvWdcD>G`CRS|*VT2`)eV=A&*xY#pH7e!_gmPS-UkJ-%Cj4lW9yY;vC8x8 zCD(eT3t`vI79CnS5;L@I7+zX8yc9EZu1&5R1`}lA-O9Uewz6}rJ67HwD?XhdixQdx zKQ^!pfcsd?d~U-$v2LD-nO|nzSJ%x~6PkVZyA#f-oAoq4wJQGU^4 z&He88`v0>3Uk|J_Jgj;tmftZuj2{I_WaoiYQ)$r~dUp;k>K>{NY~~r4WJ|BE*2VLB zZ>i=)b4L^TW$#RbaMEw&)vV{$EEmS|nm|JoG%QDB1#NShPc?=GUqVx`Se(!lE*`yi zV)4Y%<&EO{_2T+vU#$3eLQ}S+crR-yYq@E|*tBkJVw<~TMr%S-yf}965?odkZ%jEQ%%m7TWTo$fB(-(5n&99{uft19YTxS|GBQ+ zcZzsuDh!CU4^PO!|A)%LfLy;p$ie?_r_}zFlK-yisPP|@{Et=<@{h?NiX$EH-X6@) z9I)6gorE`%%$5Aa7M;6?XKiOf%$E>i9;2Tcm?^GR77Bt4~u4Ml?a2J+B6nV&JE=>tL?O@AE@ zPBk(EdtKAkn{d54XTURaIpAd6c=Lma1jG2zLHzy%^G8$ynuNpZe;8KvnD+^Mw}zjZ zknPg}cf{v8$+SWF@Sg}3fe*KEMWWvl>faK|-x4YufN<__i9GN>A?jm9{pWIchWB$7 zyzlyXAw0+VxdGl@+|r^q@7ARJV|MJ7YstWCU*q4%+iF6O+N_HCCvQG4hGz{vC*dCb z=UTYT3}Q?spM5v7`yE5K4>x2=}HmW#}!Va!6dvy%T_&gbDQWr5M1; ztXnR`z>CfI+TLw@NLC}5oIBHSXX?u>89;o2@AG|Fl7-)y$nC0@e0WlX{8Rh7DkL9O I$RPZG08C61G5`Po diff --git a/scripts/api_crawler_smoke_test.py b/scripts/api_crawler_smoke_test.py index fd5c618..2fe0b93 100644 --- a/scripts/api_crawler_smoke_test.py +++ b/scripts/api_crawler_smoke_test.py @@ -12,34 +12,6 @@ import urllib.parse from typing import Any, Dict, Optional, Tuple - -def _env(name: str, default: str) -> str: - v = os.getenv(name) - return v if v is not None and v.strip() else default - - -def _env_bool(name: str, default: bool) -> bool: - raw = os.getenv(name) - if raw is None: - return default - v = raw.strip().lower() - if v in {"1", "true", "yes", "y", "on"}: - return True - if v in {"0", "false", "no", "n", "off"}: - return False - return default - - -def _env_int(name: str, default: int) -> int: - raw = os.getenv(name) - if raw is None or not raw.strip(): - return default - try: - return int(raw) - except Exception: - return default - - def _json_loads_maybe(text: str) -> Any: try: return json.loads(text) @@ -158,28 +130,26 @@ def _parse_trigger_type(value: str) -> int: def main() -> int: - base = _env("COVERIT_API_BASE_URL", "http://localhost:3000/api/v1") - password = _env("COVERIT_TEST_PASSWORD", "TestPassword123!@#") - name = _env("COVERIT_TEST_NAME", "API Smoke Tester") - email = os.getenv("COVERIT_TEST_EMAIL") - if not email or not email.strip(): - email = f"api-smoke-{uuid.uuid4().hex[:12]}@example.com" + base = "http://localhost:3000/api/v1" + password = "TestPassword123!@#" + name = "API Smoke Tester" + email = f"api-smoke-{uuid.uuid4().hex[:12]}@example.com" - project_name = _env("COVERIT_PROJECT_NAME", f"api-smoke-{uuid.uuid4().hex[:8]}") - project_description = os.getenv("COVERIT_PROJECT_DESCRIPTION") + project_name = f"api-smoke-{uuid.uuid4().hex[:8]}" + project_description = "A project created by the API crawler" - target_app_name = _env("COVERIT_TARGET_APP_NAME", f"target-app-{uuid.uuid4().hex[:8]}") - target_base_url = _env("COVERIT_TARGET_BASE_URL", "http://localhost:3000/health") - target_version = _env("COVERIT_TARGET_VERSION", "0.0.1") + target_app_name = f"target-app-{uuid.uuid4().hex[:8]}" + target_base_url = "https://the-internet.herokuapp.com" + target_version = "0.0.1" - poll_interval = float(_env("COVERIT_POLL_INTERVAL_SECONDS", "2")) - poll_timeout = int(_env("COVERIT_POLL_TIMEOUT_SECONDS", "600")) + poll_interval = 2.0 + poll_timeout = 600 - pickup_timeout = float(_env("COVERIT_WORKER_PICKUP_TIMEOUT_SECONDS", "30")) - precheck_api = _env_bool("COVERIT_PRECHECK_API_HEALTH", True) - precheck_target = _env_bool("COVERIT_PRECHECK_TARGET_URL", True) - min_states = _env_int("COVERIT_ASSERT_MIN_STATES", 0) - min_transitions = _env_int("COVERIT_ASSERT_MIN_TRANSITIONS", 0) + pickup_timeout = 30.0 + precheck_api = True + precheck_target = True + min_states = 0 + min_transitions = 0 print( json.dumps( @@ -262,23 +232,29 @@ def main() -> int: ) crawl_config: Dict[str, Any] = { - "maxStates": int(_env("COVERIT_CRAWL_MAX_STATES", "50")), - "maxDepth": int(_env("COVERIT_CRAWL_MAX_DEPTH", "3")), - "includeUrlPatterns": [p for p in _env("COVERIT_CRAWL_INCLUDE_PATTERNS", ".*").split(",") if p.strip()], - "excludeUrlPatterns": [p for p in _env("COVERIT_CRAWL_EXCLUDE_PATTERNS", "").split(",") if p.strip()], - "enableSemanticDecisions": _env("COVERIT_CRAWL_ENABLE_SEMANTIC", "false").lower() == "true", - "headless": _env("COVERIT_CRAWL_HEADLESS", "true").lower() == "true", - "timeoutSeconds": int(_env("COVERIT_CRAWL_TIMEOUT_SECONDS", "60")), "crawlerSettings": { - "defer_destructive_actions": _env("COVERIT_CRAWL_DEFER_DESTRUCTIVE", "true").lower() == "true", - "destructive_keywords": _env( - "COVERIT_CRAWL_DESTRUCTIVE_KEYWORDS", + "headless": False, + "timeout_ms": 30000, + "max_states": 50, + "max_transitions": 200, + "max_elements_per_state": 5, + "max_select_options_per_element": 2, + "max_action_repeats_per_url": 1, + "action_retry_count": 1, + "replay_retry_count": 1, + "popup_timeout_ms": 3000, + "dom_quiet_ms": 400, + "dom_settle_timeout_ms": 3000, + "use_dom_quiescence": True, + "page_load_state": "networkidle", + "click_non_http_links": False, + "defer_destructive_actions": "true".lower() == "true", + "destructive_keywords": "logout,log out,sign out,delete,remove,unsubscribe,cancel,checkout,pay,purchase,order,place order,reset,deactivate,terminate,drop,empty cart,clear cart", - ), }, } - trigger_type = _parse_trigger_type(_env("COVERIT_CRAWL_TRIGGER_TYPE", "MANUAL")) + trigger_type = _parse_trigger_type("MANUAL") status, session = _http_json( "POST", @@ -323,7 +299,6 @@ def main() -> int: pickup_deadline = min(deadline, time.time() + pickup_timeout) last = None - picked_up = False while time.time() < pickup_deadline: _, details = _http_json("GET", details_url, token=access_token) status_raw = (details or {}).get("status") @@ -343,7 +318,6 @@ def main() -> int: last = snapshot if status_value in {"RUNNING", "COMPLETED", "FAILED", "ABORTED", "PAUSED"}: - picked_up = status_value != "QUEUED" break time.sleep(poll_interval) diff --git a/src/mappers/crawlSession.mapper.ts b/src/mappers/crawlSession.mapper.ts index 5d88d9c..52b4b72 100644 --- a/src/mappers/crawlSession.mapper.ts +++ b/src/mappers/crawlSession.mapper.ts @@ -2,8 +2,8 @@ // Proprietary and confidential. Unauthorized use is strictly prohibited. // See LICENSE file in the project root for full license information. -import { CrawlStatus as PrismaCrawlStatus, CrawlTriggerType as PrismaCrawlTriggerType } from "@generated/prisma/client"; -import { CrawlStatus, CrawlTriggerType } from "@models/crawlSession"; +import { CrawlStatus as PrismaCrawlStatus, CrawlTriggerType as PrismaCrawlTriggerType, type Prisma } from "@generated/prisma/client"; +import { CrawlStatus, CrawlTriggerType, type CrawlConfig } from "@models/crawlSession"; export const toDbCrawlStatus = (status: CrawlStatus): PrismaCrawlStatus => { const key = CrawlStatus[status] as unknown as keyof typeof PrismaCrawlStatus; @@ -32,3 +32,44 @@ export const fromDbCrawlStatus = (status: TDbStatus): export const fromDbCrawlTriggerType = (triggerType: TDbTriggerType): CrawlTriggerType => { return (CrawlTriggerType[triggerType as unknown as keyof typeof CrawlTriggerType] ?? CrawlTriggerType.UNSPECIFIED) as CrawlTriggerType; }; + +export const toPersistedCrawlConfig = (config: CrawlConfig): Prisma.InputJsonValue => { + const settings = config.crawlerSettings; + const input = config.inputDefaults; + + const crawlerSettings: Record = {}; + if (settings?.headless !== undefined) crawlerSettings.headless = settings.headless; + if (settings?.timeoutMs !== undefined) crawlerSettings.timeout_ms = settings.timeoutMs; + if (settings?.maxStates !== undefined) crawlerSettings.max_states = settings.maxStates; + if (settings?.maxTransitions !== undefined) crawlerSettings.max_transitions = settings.maxTransitions; + if (settings?.maxElementsPerState !== undefined) crawlerSettings.max_elements_per_state = settings.maxElementsPerState; + if (settings?.maxSelectOptionsPerElement !== undefined) { + crawlerSettings.max_select_options_per_element = settings.maxSelectOptionsPerElement; + } + if (settings?.maxActionRepeatsPerUrl !== undefined) { + crawlerSettings.max_action_repeats_per_url = settings.maxActionRepeatsPerUrl; + } + if (settings?.actionRetryCount !== undefined) crawlerSettings.action_retry_count = settings.actionRetryCount; + if (settings?.replayRetryCount !== undefined) crawlerSettings.replay_retry_count = settings.replayRetryCount; + if (settings?.popupTimeoutMs !== undefined) crawlerSettings.popup_timeout_ms = settings.popupTimeoutMs; + if (settings?.domQuietMs !== undefined) crawlerSettings.dom_quiet_ms = settings.domQuietMs; + if (settings?.domSettleTimeoutMs !== undefined) crawlerSettings.dom_settle_timeout_ms = settings.domSettleTimeoutMs; + if (settings?.useDomQuiescence !== undefined) crawlerSettings.use_dom_quiescence = settings.useDomQuiescence; + if (settings?.pageLoadState !== undefined) crawlerSettings.page_load_state = settings.pageLoadState; + if (settings?.clickNonHttpLinks !== undefined) crawlerSettings.click_non_http_links = settings.clickNonHttpLinks; + if (settings?.deferDestructiveActions !== undefined) { + crawlerSettings.defer_destructive_actions = settings.deferDestructiveActions; + } + if (settings?.destructiveKeywords !== undefined) crawlerSettings.destructive_keywords = settings.destructiveKeywords; + + const persisted: Record = {}; + if (Object.keys(crawlerSettings).length > 0) persisted.crawlerSettings = crawlerSettings; + if (input) { + persisted.inputDefaults = { + field_patterns: input.fieldPatterns, + type_fallbacks: input.typeFallbacks, + }; + } + + return persisted; +} \ No newline at end of file diff --git a/src/models/crawlSession.ts b/src/models/crawlSession.ts index dc22467..e7610ec 100644 --- a/src/models/crawlSession.ts +++ b/src/models/crawlSession.ts @@ -23,8 +23,12 @@ export type CrawlConfig = Plain & { crawlerSettings?: CrawlerRunSettings; inputDefaults?: InputDefaultsConfig; }; -export type CreateCrawlSessionRequest = Plain; -export type CrawlSessionData = Plain; +export type CreateCrawlSessionRequest = Omit, "crawlConfig"> & { + crawlConfig: CrawlConfig; +}; +export type CrawlSessionData = Omit, "crawlConfig"> & { + crawlConfig: CrawlConfig; +}; export type ApplicationVersionCrawlSessionsResponse = Plain; export type CrawlSessionByIDResponse = Plain; export type StopCrawlSessionResponse = Plain; @@ -32,46 +36,91 @@ export type GetSessionsQuery = ZodInfer; export { CrawlTriggerType, CrawlStatus }; +function normalizeCrawlerSettings(input: unknown): unknown { + if (typeof input !== 'object' || input === null || Array.isArray(input)) return input; + const src = input as Record; + + const mapped: Record = { ...src }; + const remap = (from: string, to: string) => { + if (mapped[from] !== undefined && mapped[to] === undefined) { + mapped[to] = mapped[from]; + } + }; + + remap('timeout_ms', 'timeoutMs'); + remap('max_states', 'maxStates'); + remap('max_transitions', 'maxTransitions'); + remap('max_elements_per_state', 'maxElementsPerState'); + remap('max_select_options_per_element', 'maxSelectOptionsPerElement'); + remap('max_action_repeats_per_url', 'maxActionRepeatsPerUrl'); + remap('action_retry_count', 'actionRetryCount'); + remap('replay_retry_count', 'replayRetryCount'); + remap('popup_timeout_ms', 'popupTimeoutMs'); + remap('dom_quiet_ms', 'domQuietMs'); + remap('dom_settle_timeout_ms', 'domSettleTimeoutMs'); + remap('use_dom_quiescence', 'useDomQuiescence'); + remap('page_load_state', 'pageLoadState'); + remap('click_non_http_links', 'clickNonHttpLinks'); + remap('defer_destructive_actions', 'deferDestructiveActions'); + remap('destructive_keywords', 'destructiveKeywords'); + + return mapped; +} + +function normalizeInputDefaults(input: unknown): unknown { + if (typeof input !== 'object' || input === null || Array.isArray(input)) return input; + const src = input as Record; + const mapped: Record = { ...src }; + + if (mapped.field_patterns !== undefined && mapped.fieldPatterns === undefined) { + mapped.fieldPatterns = mapped.field_patterns; + } + if (mapped.type_fallbacks !== undefined && mapped.typeFallbacks === undefined) { + mapped.typeFallbacks = mapped.type_fallbacks; + } + return mapped; +} + + export const CrawlConfigSchema = z.object({ - maxStates: z.number().int().min(1).max(100000), - maxDepth: z.number().int().min(1).max(1000), - includeUrlPatterns: z.array(z.string().min(1).max(2048)).max(100), - excludeUrlPatterns: z.array(z.string().min(1).max(2048)).max(100), - enableSemanticDecisions: z.boolean(), - headless: z.boolean(), - timeoutSeconds: z.number().int().min(1).max(86400), crawlerSettings: z - .object({ + .preprocess( + normalizeCrawlerSettings, + z.object({ headless: z.boolean().optional(), - timeout_ms: z.number().int().min(1).max(86400_000).optional(), - max_states: z.number().int().min(1).max(100000).optional(), - max_transitions: z.number().int().min(1).max(1_000_000).optional(), - max_elements_per_state: z.number().int().min(1).max(10000).optional(), - max_select_options_per_element: z.number().int().min(1).max(1000).optional(), - max_action_repeats_per_url: z.number().int().min(0).max(1000).optional(), - action_retry_count: z.number().int().min(0).max(100).optional(), - replay_retry_count: z.number().int().min(0).max(100).optional(), - popup_timeout_ms: z.number().int().min(1).max(86400_000).optional(), - dom_quiet_ms: z.number().int().min(0).max(600_000).optional(), - dom_settle_timeout_ms: z.number().int().min(1).max(86400_000).optional(), - use_dom_quiescence: z.boolean().optional(), - page_load_state: z.string().min(1).max(100).optional(), - click_non_http_links: z.boolean().optional(), - defer_destructive_actions: z.boolean().optional(), - destructive_keywords: z.string().min(0).max(5000).optional(), + timeoutMs: z.number().int().min(1).max(86400_000).optional(), + maxStates: z.number().int().min(1).max(100000).optional(), + maxTransitions: z.number().int().min(1).max(1_000_000).optional(), + maxElementsPerState: z.number().int().min(1).max(10000).optional(), + maxSelectOptionsPerElement: z.number().int().min(1).max(1000).optional(), + maxActionRepeatsPerUrl: z.number().int().min(0).max(1000).optional(), + actionRetryCount: z.number().int().min(0).max(100).optional(), + replayRetryCount: z.number().int().min(0).max(100).optional(), + popupTimeoutMs: z.number().int().min(1).max(86400_000).optional(), + domQuietMs: z.number().int().min(0).max(600_000).optional(), + domSettleTimeoutMs: z.number().int().min(1).max(86400_000).optional(), + useDomQuiescence: z.boolean().optional(), + pageLoadState: z.string().min(1).max(100).optional(), + clickNonHttpLinks: z.boolean().optional(), + deferDestructiveActions: z.boolean().optional(), + destructiveKeywords: z.string().min(0).max(5000).optional(), }) + ) .optional(), inputDefaults: z - .object({ - field_patterns: z.record(z.string(), z.string()), - type_fallbacks: z.record(z.string(), z.string()), + .preprocess( + normalizeInputDefaults, + z.object({ + fieldPatterns: z.record(z.string(), z.string()), + typeFallbacks: z.record(z.string(), z.string()), }) + ) .optional(), }).loose() satisfies ZodType; export const CreateCrawlSessionRequestSchema = z.object({ triggerType: z.enum(CrawlTriggerType), - crawlConfig: CrawlConfigSchema, + crawlConfig: CrawlConfigSchema.optional().default({}), }) satisfies ZodType; export const AppVersionParamsSchema = z.object({ diff --git a/src/services/crawlSession.service.ts b/src/services/crawlSession.service.ts index 8df5da3..e38fba3 100644 --- a/src/services/crawlSession.service.ts +++ b/src/services/crawlSession.service.ts @@ -10,6 +10,7 @@ import { toDbCrawlStatusFilter, toDbCrawlTriggerType, toDbCrawlTriggerTypeFilter, + toPersistedCrawlConfig, } from "@mappers/crawlSession.mapper"; import { type CrawlConfig, @@ -27,6 +28,7 @@ type DbCrawlSession = Awaited ({ id: session.id, appVersionId: session.appVersionId, @@ -34,7 +36,7 @@ const mapSession = (session: DbCrawlSession): CrawlSessionData => ({ triggerType: fromDbCrawlTriggerType(session.triggerType), crawlConfig: (() => { const parsed = CrawlConfigSchema.safeParse(session.config); - return parsed.success ? parsed.data : undefined; + return parsed.success ? parsed.data : {}; })(), stateCount: session.stateCount, transitionCount: session.transitionCount, @@ -85,17 +87,7 @@ export async function createSession( crawlConfig: CrawlConfig, ): Promise { const parsedConfig = CrawlConfigSchema.parse(crawlConfig); - const persistedConfig = { - maxStates: parsedConfig.maxStates, - maxDepth: parsedConfig.maxDepth, - includeUrlPatterns: parsedConfig.includeUrlPatterns, - excludeUrlPatterns: parsedConfig.excludeUrlPatterns, - enableSemanticDecisions: parsedConfig.enableSemanticDecisions, - headless: parsedConfig.headless, - timeoutSeconds: parsedConfig.timeoutSeconds, - crawlerSettings: parsedConfig.crawlerSettings, - inputDefaults: parsedConfig.inputDefaults, - }; + const persistedConfig = toPersistedCrawlConfig(parsedConfig); const newSession = await prisma.crawlSession.create({ data: { diff --git a/src/types/crawler.ts b/src/types/crawler.ts index 447353b..efea897 100644 --- a/src/types/crawler.ts +++ b/src/types/crawler.ts @@ -3,26 +3,26 @@ // See LICENSE file in the project root for full license information. export type InputDefaultsConfig = { - field_patterns: Record; - type_fallbacks: Record; + fieldPatterns: Record; + typeFallbacks: Record; }; export type CrawlerRunSettings = { headless?: boolean; - timeout_ms?: number; - max_states?: number; - max_transitions?: number; - max_elements_per_state?: number; - max_select_options_per_element?: number; - max_action_repeats_per_url?: number; - action_retry_count?: number; - replay_retry_count?: number; - popup_timeout_ms?: number; - dom_quiet_ms?: number; - dom_settle_timeout_ms?: number; - use_dom_quiescence?: boolean; - page_load_state?: string; - click_non_http_links?: boolean; - defer_destructive_actions?: boolean; - destructive_keywords?: string; + timeoutMs?: number; + maxStates?: number; + maxTransitions?: number; + maxElementsPerState?: number; + maxSelectOptionsPerElement?: number; + maxActionRepeatsPerUrl?: number; + actionRetryCount?: number; + replayRetryCount?: number; + popupTimeoutMs?: number; + domQuietMs?: number; + domSettleTimeoutMs?: number; + useDomQuiescence?: boolean; + pageLoadState?: string; + clickNonHttpLinks?: boolean; + deferDestructiveActions?: boolean; + destructiveKeywords?: string; }; diff --git a/vendor/coveritlabs-contracts-1.6.1.tgz b/vendor/coveritlabs-contracts-1.6.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..bf90dcfdba49bff8f686dee6e30b2387c99c5b01 GIT binary patch literal 25449 zcmV)CK*GNtiwFP!00002|Lnc#a^pC%C_KL%QSX4`j+j%`?XtCaDOa7E9+IMDTDELS z);clAO_2~qi;J3zWO=3|?yvg*U)(1=Px1voa9>0%6TrHc~yOW`lC z@Xt0a-uO$Qumd6sg#z&Y`$qmTcU`u{J%AgEZiD;Q-F=H=AE?#>iodZUnLxZ!x4L>rE0i`IQg({e!1x;wop?HiMHtrrchW!j4 zCb9`keQYgw0|H(Xfo!1U%+Y7p8 zTC!zZsJ%9pMW|Uaui!G&(vYFW8%tmrya<5_1p`_p#;L`&A6gvqz}+?4pWhu6k3mBR zGv2U(J_c40(IX&hf?|uZCIxXl0-`S1D%33Al6CC}=*GJP003{Ax*+ow6oJZH(2#k> zJOV1e1S8aNCPVQE7;t9jqAj4`sxHc7{M9me%~Vj+BL|eo&(I%0lugT!N4DkFQf136 znJN4d{hUE#tQ#u+X^#}yoVailS%8}9wH-slK!2FJA|kYbZ9)-PIuH!VTM&rQkmtOG zxWCYisR{n)|NK87YZ%mGqAL)!ku`@~sBE+ZytV`?T#110*;tkASNc%n7e37>WTAi_=v?oP-M3W^O3%mTi50 zckpns+&1ckfh9@q+a zxC^vKJ>Ie{=89@OP&9YLJq!TVsDLW>&_FP&cP&7Hd#d%oF`XN#_3q%l)4IQFv&VpI zfW{qQyKLhDw673SA$?9G?M=})tOKJl02QvqG7m@Z4!8#SiO}RKY~z8tIRfo_mf_G3 zw$I+)-%zc=5h97XYqZ&BhiyD?)D5Umw^WU7zdN8>TpN*x*mqaOv^P-Q28*>SW-WNQ zVnL_Pq6(qGz+**eOE8yp1uIZT-k}MB1r5~%3t3Ts0Zm)6kj%-d3Pq$=aj~qqZQdR5 zj%py{iR6bUThksrLBYbBO`glnsvsCLYOR17)Of{`p$TkFgvPrAeuh*MKT;r2_ytC= zsvAUkq8To0zuU4QLoC#q#S6|H!xDB<|7XPhLkrA_ zfaa2Hoy-g!@$F<@JRv+myndXGj-Sk3+keI4#pzkp{<|oZPWSfTYy3O3O$d-}Ay|iB z4o*(M-zBI)16wbnCDOE(E_{@r_7R$3UOFxoj|+H#L2C`KLJJz;(BcgVT3;>mFalw0 z=qeb?3bK2$MC%y$1Kty9bZZ}z+Q;+a5ipmU#Xmb3l|LMu{0yL}$eQ(0lubNme?B?5 z!}I-EgkxEQ_hglL51(hcftHuU$46-1>Y}Z{!z1u7Fy|E;eg>9dL-5CkF9)RJaSKWY zG)-C87+r)T-h?e^&X9f57A8>T!B^l`eZZtsJ}F&jpoaMyq}F7xF_yo6XD0F%Ov3$l z8=6)M{B`T66P(=2!771@wK_?wj%4eRNXsLUmdAPVF(6yuE12uD2nt^g@MZzq?h@4I zrN<)p3Ji#JI(&cl_mMpw*|H*@U=bVH;}fsu;RmBv1djt`K0uSfR#!cmY|+zNI7;r$^ax1L7jp(3bC;igP4 zN2QrEC^zTg6FVRD>BXp~JPrD-*{F1C3s={3vG%FWs*Sm*c9(p=iNEn`_eo^VO1JIn zJ~KJ12x@mC*1DZR|5_XNTeNURn|!Y^5o^kPq+BaPzpDu4X1lu3*^yechAoNaC{|}W z-1RS0t|my$Ce@lu*~y~L)UGE&sXG??*Jh34B&sc4Hznh|DTzzEMbWieiRLt+ex-C5 z!&sdT?n=3*`QP{@V!gBqjsOgUiU?$If0Q{_Mlf(Y7|T0Y3EI8xTI-{ zuGLx`#Yr^P;wFqVkq7;T(l2+HBU)kzj`d%r+?618npEqFqdF2TQ!L$-ggV{5#BiIE zS!cSJep}|c!pPkfd>JZ&)|ib-XYFCXS{(Kp1$Ls>>9YI%bU?K!x=tw!GoCTD#<48L z4sPXohidBMX8Bqf^jk_t?P`MBeGy8^d^By$hqW%oBS8OzkGb3s!w$AX=l*6C7p!FMl>VShTHmIUYf8AJEE3aZnl7>;UI8Aj5% zzF${62TQ8TQMyD^9j?Vl zm*Y5IU50Jy_x*Cb@1~^Jnd;?4s3~K<)IH$kW%BVEsiG?C#Jch5+oho3vPoFdh0N`_pO>G?=&D6FK{DOH)e z`hlGc>J@dkL~~m#T`u|JCD9+8U!Evax43*D4HtrXSs1zgjNnmgL|LcAOB2S zCsb&f$%fI26%(x+pkqSto8#MgpJ##n@LRwc!~+d3Tm-_)P|n3Vx(*Ro&Qi3-l6C|N z8}KLF0EzJ@kG+zYKRIom-SMZmJNXHlb$P`o$r#*Hno{h`-7&4SG1F3Q=M;9Wu(j7? z)M~?hcPuJzJn!UMT``E_@2N4T}}j%+V?$D*?hSd^bRJ7UXg zGUR+042d`#R`Vs%L$NV?()=B}*?U0%z72kQ0k_)>9^*5scZY*k!HP8J5$8s^Br)LOeY$+$iXA8k8=r zQSZ`5<8q^PUxxkb>ybK~v+|-&Yfc}+Gr{eJ6Vlr{bvw1{OpvdejC2{K!NaGr(62^4 zF(37c^O5X$#&BJQp78x^nbzp#sN9<0 zE|9axpG0BVb|w&#Zd@a`zlf;Ru!e}4LJ9gKul19xde%31f|GX*e~(LFP(8-Ti|fw3 z)Fo=&ZHX(cOI+d#5u-9=9mz{>_ENMh-KBNuPFg+_%=ol5HopO%u}W`Cba{QEgC<1+ zxp9jlKcR`%%C;n)Za7&}_GJkc*P7FH>)ltFzO#*oFm>mdU~{fLh&!h8JauR3Z3b6t zDfj>0(7(j}zhLkS1scfWCO^0G0%Y$0OXXrb{_DJadcNQP{~7oHanVZi0=UJm>TGr5^l7TmF0`phYDT!$IKV16Zf$yHJ-#$B_)rsj%h zvoJ1cAzFT&3UBer6&Q3ILyDDg+RU~paLn6^Wv0|&oRM$YS{qs@-rmgiWw<(OUUYxH zotUq*`H|yz~U2W1Ua;ucxy*im4gkHd98LS%dy$ERn{7{MghfG)u;~X8^sCF1sYU5a;#zKdqUiGwJZYZN# z_erc?&V?%!LwEMlFVl*kT%zyRuyFXpYt~O}{gf>&c*uaTp6~w}4_)2Bpj5NNWxl zhO|ZTQWKr6)tWlRN>>ga^*W1VTb?Jdi~4RYVfXTBRPNd$EnO2{;+V0T)Eo>MiE2|a z#=XAKn?mh!A@=F9SW~QF@3JUXFD>C&s*`rNzf8I7nj{RQ8+ll7Bf3@G_FB?!hqX(q zU#=GUo+~G2WXzwamnFuvj&XgW>eXIrKI&e+4EwFZsMd}4dz;>`?ZkO4^fvM^^-CDe zv)EG!LcPA&#+AP(0dHc$Qkx_%)dkIhMomYd4IfjQ#oBU@A@hrjKN& z*PV;~)_Bw>-<SXQ83=MLGRfdsMROF zygnRmM$wF9IcqN4n2!+epRXxqLVL6ld7F(?@r8KOZ*Jv5247qo;JPjL&K{7?6%eOhs7=`mE3Yf>$)$Vksl z|JBPUB8LOIO->1xp;H3aMzh5y)^0j|xw6~7qVyYL4|!`jEjw#DeKKQcT4(AM)#jRd zQ@pyKjI_p#*bwgeeMQ#8t6!!ZTa!4QqPY9mkS8Yg7=Q7E_V*+kR;{8>z|*PiSdQ zsCq@YR0QpME-07Qu-|wY^hAZ5Ni;K2+!+0+jQS^F$+AIzOWE*Fkn+ohJg3g7Y1t#-QI#<-(&?;fVgUF(@*(7lxEGzO#cP$)JZ*lq{Ic}BKMBL=56tzJK{ zU6!Sq?HZ+bn#F6nTX-IKiY>ZZ?LPDiRpz01jbYHOWEdT`+U*n&j6u6f>CHHdvugL5 zx~dyG?XNfCd57+`TzNFK+U<^b)#_f4yH%yx>L&56P_K1T>o=O6vu?AqXyUbM^6cW= z=}eWYR-xPO7hA&)+Z}fb4X)K`bsZjXNH>cv{@v>Jb*ms&JIZCJ*QKldLgU)+i$gqx z%w(1dyIbfoZT9lM{Y-yY&WDAHy!d6l4B0PwFdc{kXYK3#DHTnna-H!b1h<%{HD91xV#a?DNf&% z(DPLH@OnY#SzhMdFynLsX5{po$5^wL$2~8j)^+3WhHYnC&YIxKMCR;?A7N}VixLk) zcEgGD4LFf@m=lk)$~^VS9dfnm>}WTf2!!9C1plEAtY)ibQVIGNJqLuEt)>eQJ{iOG z&3OX^zI{1(Kejai-)Vmz`2GR>3nd5w7&r7UrAKnm$F!*4&13t4dg!ztI}M!arpmDu z@D-gBIRH-Ze*=B|Bl!C;(CYA?s2%#h(AO^;W7O=h9gIvdCo)b0j7JgUK|XAbLaWng zpmaW8K`AE=qyowUcYA-sK44yybK>E(!I+dWCjPh0aj8<=4MyZNCn8P* zjKwL&f_&H<1&Y3FIm|fAi34eYkvPLhxL-EM;-2cXF%suFk#HJd9L_Nga z3NCW?-(Y*V^9$TL`s>TVA0LA11N{5n;C=GpeE6?_{qZ4fEy$tKk`}LUfIw7BBcPja zW!6%&PT>_%w&?X(`s(hk zo}$yUaU*R5>fQa~E0EQ-$H^@J`d6IP@;Fld z0joI@d`UK~zXH!@c&{G;^$1Llz_jFA3-TDepO(PauU~QIO%$&wegNnTm^w-s;UPj( z0AB%tEJ5pTp}Aq5j%u7)y73+X|M10w`yaR&Lg`R~)*(t9iog5{{zED}o|Ngi00Vk1`Yy*A#;35oMWd4QKFvO)!?C24Yp!FGeoIKaz_Gq{d=Z(F>PFtUCj9##(p z^~jC9bTL4QlHVhUGsI%`14d~E{^RRca40&y>q7+iKM27%s1e;+JD%`nU?h(PUQzrg z-Pwl^Q33P|RaE}v;Eylc#`*>ceeXwerrm!({NR(nx^dDS*R|Rw1>dg+A~k>O#$en$ z1bIOu2gC+jCRz^e0<yelfDrK=%7zjtnl zSIycRlIVOHdwn+Ly}=R5)*HySnRJ5#efxdbYZUMe4)wNPW!(&;E}TG~hlx<9Yw6ik zhcqWMl7%A7p9ED|;Hbh$G#c}Ea7i_s2*3_^$4JlPVXgZzC|}Qg4=n?%h8q6P1p6(^f(m7y1U)y>N4Ld_N(>PL& zdz#iOouV@8T`NK$|3*!U=HE~l=48e&S6mJArAE=Lw5Zcp*Yi=$i{Ot@nTpd%^s ztDs(*Ugp;}cgjp!?P0HoaB%y33FPmP7-^`CdezzJ3g7UlAYQQdGo~`+XhxD~w$_wU z6zG6rWp1H&y$r>{aZbs&%Bh8Vr5KdU)J(6pel}+}8|!VIiHxV4iItVjI$a|C&~8!m zUC?^pv!na5F|~p$Pt^7s)p%=Qqe+Lvk{Cg%2_g1 z?4kN8&g&jmBbl{d*2wsA1ioz^*|4@x2?I%564r|VNm*33N{~i=Zj&W;1@-Bx=6ZQ4 zzbW;b=w~Nj1!NL-gjEShL4C)#zML zvcKhcpKg=~GH!4u^Y^=sp^!Fx;d&e}TW-SeQnfA`V@1Fl9ww2!MP;k}zujeuU-`PcR|NiTrGyh9i zv{IA5;A*e!`(q75H&UHnqZjgm-$o@|@+!L6S5k zSqVsDh-T(kEe5lXko=dIrQ7yp0o}+FmY(0Lm9mePZx=cj+)Ky&7{UCg>A6);8A%D` z+~m+LvfDwg@q}*YMRSp|lTN+TzgC7Q3r1-&7HgNruy^YDb~s7De{gevhF+2RgnbR} z?N#@6o))DyTK((gsC*stYcv?C>)cA6gkG`nZm5R*8&0N)qEJM)L{Y{GCmTxe)>rPh zb$5n&(iz_qgv2)YMj$He8-$@peIAANsq$uHJRyE~p61ScADiAgAgs7PVG%WyVC^NS zsr+VNvX0vUsipNv4e8?;p$XbLFPX`Teg{OB(=6P8A@xJqw;}s4=@4WG;b58eU-|U( zyd1UvPS4Bx{C|Hc`!6P1QOnQ$9ya(&-`3zWYy}3>@e!j+41AI$O%Hz|XaqmiHGQGA zpah>|CZ1muG!{6lzp6wN2xx6=);WX{|dOY0a10 ziBu&tbwe~M-D_N~To*^R?n|4Z5dUhT*Vlmb)d}_0?KtHTUcB%aZ;s?ZjPc?jP8Y(l zfp9D}Xo*Y45#>H>mKs$b$9hx08mh|au!m{8P9Edo4A#2SaDUS}U2?uRo|*&39WyzE zXN~Vqlwp`Y1LxxL`@A82_9M8GwztcawusK#?6#qPpBSknk4l#&$$5BlJGl*wMN_?; zkE-GEL%HNo{xFjNFgAW2QXc3B$-fNyEyc@w7?c0&GnDT$f}1kZx>k**5^-zw&WfXM zVB8iuC#c;8j&|ua2dvBgWr+W6l2=n12I%6NdS%qBE^4$pXSh&b554LlfZK$T8k3PW z9aI^o4>ml;^hi?za$(k)TB|6iPJX5u-64GlVS28*OXm&lvq0NIo$2FzgeW(lCD2lj z@NWoT40M74&yS(rY92vd}Lnc8wBH9zVu;O-9j1 z=;fO3r84T%1=u2U6lF7O>Nl<0WpSWBE3WQEUY|kf(iY0DCy2#;jjr6+aM(7p&?}=W z&r?D>^oXurqaAugL=V`t@KM> zOSo#Ck$hHoBKTe<;Y&Z~$2W_=bcDZjl)uyU z{FPYr8e7C3O44PYBD(BTRDSOx@_QeZ-%3Q6RT6dCrxo&xq9uzDE!R#<*ALBzerQJZ z0~^r~Y*atcR9HXIRFZyZX6pxzXC#F|wqZJNCHbUzTW5-mf}kq)u$W5IHK9QnO{52+ z)9|SW-#Z(RYUmWk)rI&p1?84D@#)Lb(RXZ9zee@?CC5&z(M_lRgsn5Z;%uZ`y7r~3 zGh2+LIC-Nrr7-N@db$(6<|eqge({*%Qy4U+3qkFwd|x7aj49@uQ0}V8zIOAYvJwBcwm64u*@~pIe4j>pfqTE+X`QYq3-M{-y(f|=0eVei@Omn?f! z7I9AQVX1psqa~87H?qepoy|p*OS2~`5W$@^(f$p|){{s}L)>2l*|XQhVai%ubNXj0 zlwKE0U2{~uG)C%Wc_iOlP))tw#In#xwu2z(eS4>|ooN+?A zrDr1*<>tM#M^{s#izQD_QN(r)>!~KvQ&mnrtykEXPX$GUzwjj_`&h?tcwS!N==YhR zcBe$A5Zj>`+xG=@$4snsEhj2JtULIAOBwau*Bu=@OGNpN`(1ba;8^(<`A-VN9(ubV z*?&Uvox8Xv$}UZGs=i?B3|pKCaXc4;p0Z5BQ$R0RYCfJ=U#gd5p-fK%tr5VLtJ5`< z|NWN8rn#&uN5YkEAA(HtC3Mps>=TdOI)tB{W@c^w=$WAP-_8MbP}u1i~V%*a@J zHUiH3ejTw9Dp-fM>lc)yH)8wdXFP|N=+*>}PV`-$8_nqf`E+xf4{ua)J}QZ(=LalZ z7l-m0EB3D^&W#u3zrZ$6WsNqfYZtkEuC>p!QRz|})jGCw6W~%7dxOBf|FnJGB(_s~ zjW&{NU%CFYsXZ)R&WELE<%Ya6I?%f6&D8fnAfKPZU%|<%pR3I@I0hWK!Lphca{Zi= z^tqody_40^?dD=YX_TYTF@Sq{Aj4+;6;d|QL~acXihFM@E+WV(X> z66{}$j8*Qwklol>w=c(5TZb+z|IdX!S|iDtz&LAwCRQ))QCVWHoFa=2H@fDbOq2J4R|O*LX|TxJ5QmkiXB%z0@95B`Le*=kGQUx~wy0j7A=vP0 zY6~^76l~vny4Lu9OBZ__eg*la6skWbI6d9;L}erwlg~<;=WE|)A4EVdVD^uv4;BLGYP9`G`1%HI+_rbvlU5~>yeZ;M)6pp zwMg3OC>8Tt2kp#pYHvPJ?cI-5vA8B#sq<;`!D@el(aLWwcwulck|F_edC7^#)on*g zeq$jJ?U#`0J`xF-Ke>H8gm628@*50(WMh1AB0-P2M<*UExg9n6%>^%lGaf)6j_u4H zJRb`Q-Hw#}mO?O6HTB-Z`UE8gz-~uVerq8Vo11VoJr=o}|4w>R(C>C+laLYjA|9BbBWgigCU>5t(WtPw%t1{OoNN9MvC0wD&nWKJ% zPu0DxquGigk7Z~AUR%2OpsC0=<9z=Re;Fvy27FDoX>cdhBJD99QH zy;dLtf%#bPs0o79WD;j6HJy4*DG z{a^Cm<0n(scHPH{{4eElG4}q$+3EiN@2mX#*E?)E93pe;@H064JGt7f@FVjC*`x+9 zSmxmosWx}sqC_=|$LGgIufRR$C4~%v!dbwa@JMM<_Z;wmf&ckG{}0^bi^aMEXnQ<{ zhGQJbj{4y>5%?OydA0_@cGCZPU828GU(O}`C%-`DWsF4VO_NYFGg2%b2WUhp0#*3O zJM#DgaOxg9fB)01H?v2IY)+i#YY#tze+B9rP?Al{SfYL=mNhd!pPXnj^>{XwjwRWe z*dvtYfI$4ig=oN;ZpxO9+Xcb7q)M{oqKUuTh7#<-hkU{~!+7HiBmVu-8v%quv~Oql z%t*)XFrZ#W_ShR1@bRN#9>I^YiZv^G>_O8-2ne7PbZ6Y*uIh6i14Wm)U4Zy^un=Dl zMWxzI1%|EpU7}|yOd(X+fC}VIjGYNpUbAH3qx&sbZ)U(5Z@?f}r=MUYBu6qvOn8ZM zM@SlEr-A$ewQzycC2vj0ghI6tohQP}NB^%wzri1o-b6Tqng}&PhS33u-=IaE{CDa2 z?6`0k12eB=K#FqgM40fJ1QlJvAkU8r$K?=%5?WwtKJpr`EMK63+ZD^l1z$0wG;}1@ zsb4xS9G|bQ?^G`20Itnc7n9=g*^0{KzaKT-g62meY&s$3^tgCj2my$R8Rq_3oL9kN zS7fJJ;rKMG8kR9MOXpB?te{-YLPC|^z*p;%Y<-;Q`qU&UWnzy$2lWNfP+&*6Wc9+x8{JF(3v)}a4=d0afc2$f+ijx`OgaCZFZxa`XR zyMsU89qd#4?d1KB@cS`%pisFZXJF;^!_#vKmWsjM&bAvt@|gInM0N=^YEWX*ziO!{&{q$I=e3O@be)> zan#WrMcp0MDXMjULh)Rv3YEG}H#gST}D7E7J zMnzvZiYeBKBAnEk=pXP?XLDc!x3Z z*v8C#%B;#aAPkqssS{(=B~@f7MtKzye`hG_cP2_=p+ZrG2#L>EyBU`fLs6v|(gY8N zVvNd4q<^6)>X+EK=oO06BS@cx#)YRSK4C~KdhJSH(Plg#)JAwf$YLnwh55ETNRiDs zy;;q-%MILT9O>a};JlZmgz0C=EXKgdm1uP8<-2fBXsT|BXE0E8hPt6r%Qj z@#5lqZ~y-=_W#psTmN4uilUBWmZD@9&9*1@Xi=l61)h4(HDbVj`GTzd8GUq$)E};B zut&dCD9T2E)aIM2c~D@CHfXh`Cv#7^|`#I5QDF&tPUI-x=Ou zx8O>N=y`?}Q{2hul2x-tF;6>TpJyAO^)^4eA9G#W|ND;Xf8_n^&AfiO&wo!(OR@KV zi|2*?`v3jb|G>C!y!1zf=o`8F*tok7B{Xq<`CGQ&Cc|wx+mWM??BHBS;pxn8VwiOD zeg|(iqPrb;%N*G47;JEp;$xcbNQ=Dq*8HyRLr0I8>qJEFt3f-^Bm!Xr^#cO-hm>WI)~tR4w6K_i5D(=g~ENh3_4C?QIg z=ZOfAH4h0G9_%>ERu0FrgIokBPZlvRubgDxj$$doZkQ!d+)`B-k0!OR_)E*p2J)sg$$llV=5wefQL`y^FG6(#{+5KgGbUKqr2V6o1j9d6SE4COO08R@ zH?5nW3;uo~fv!+ZA`dBb&$4k1NkMtTkQA9lOI#Y%Fa~Ievs3FTOioE)@FKiw1GYd0 zSZo94G0fe9#b>BL+q^|TG}{&oMJ*z;ZB|V@?f=sBmT(NjCj|fLl0dKi69c>@r1t)y`ZJ7seCMP{LWIN1%mBODNB#0G=OkjQ{|VDHarDv6** zsKkgf6mzqZL^6G%!7*+)T`)L?T{ng#$w7oT5O+x6N(AX7h2VR~0@EE;yNp-!xfudj zjO(IR3LX#SDy`-_&O?=y4HuRCN|N|m6BS902pqC@MB|TK_6EFENOTYkZf7BA+YToR zkD|8@9eZMOkVuS>2#IJreHx7nyh;gtb4X-^y6|FsVN7xm$7Eu2fBrc`QRlIMEYAdE zE-EWk30ayVtRjHJs-&6`#T*m~hGNcE<4!Eu2<`-(uxtOVwB zN;vL*YnG)m29Ks&A|^8i#R_K-8Pw4M3nZeW_rWMXuK#D_Q-BrmAI13n|BGUA@BjHB z{69~x>HSGK2+(U3rDJ6>ON{&c1y?*-5%(Dn-3uH9EKjZ|YI$;%BkuFGiv2!OSFyL^ zGFseKjO>nI(p}F=LL^`O++sU;WPz#SReUXYpDIc4h%T>TyK^mgY9fA_k(HV7&J*Dg zTc<`b&z1Gy4U)ilci+%emhTBJ!EE9jt7*aXzfjaOQ$Y)Vni*B9klj2LGYaGivDc`W zRekbBBB@g$ zpb4^>A2z=@Vtp-X#Q7JRa6Zi*`<+eYeJP!2qzY3Qtg^&eNyTikbDzPK8^3hKQZIDR zwYQMtXF|JwHK`?CPT^O4s^{X@-i~{V_Me{&XQz3tvhBY@`8*c?QMxGX<3GNi{TED% zw8Ho!HlNcX`(%%zoC6jTmq64iTJrJd@EIO+K|Tj!#n9(T#rYiY>SE)tl3CmHLF_ft z>iNI(EO)Esf8i`P|M8uG{rvx~^Z#jwv;RxvAJztF|5k8y0NTHuyEA{vQj}C7GuUL& z)patHVoZy5h`T0Seq<9{yp{-5tR|09FGi`%~ekgsBvNXStHZ7*mkc!g%yQ9zrf zkI6M0JBZoYbRPL}DE_ZC&d+x+J(91)6&PgjMJ7mfCrOA_U5R^HjdRjXC6a+592~)Z zixfn;Vj?07RA^weZErk{R&vM>W%D@{*vpPPF=c2gJ|MUIU|Iny!9{&}WqStf%ZRW~5rTwM;j$)== zuf=6Qc}s+MGtr28+P_BbZiy}`>uwsqsjmp1ibgg}WMjruQzBk`qH&;bG`^fh(O6-k zPbzEga5qoH;@@!bNg{VunClQp$+shtw*MqwWQeFTyPKrB*_-hRode}0CuCA?mHFCE z$XxkXlJ0u!BB{)9Z;AYH>;E=E0W0#q6fVwV>;LK5e*gaitp881w)!V*{iPa3NmzAE z(zpIKBfh~W>u(Le`Vh!v2j1h+^;F4GN=4sWla|77~#ee3ij_~_~O;h>rNzg#LKT_4H+b-sW9^E>MQ z=@i6rs(xeg1YJ?^f`nnjQt>L-Y4P{Biv&9VGgJk5hTR&Rl z;+EX*sK-owOmsA}fqS|cJWj1-T$w`@#Iz;ej)X4^MDJ~%Y>JTT__pLC{i~!U2qI<2oAJHFM>+VN% zKcf2)-H+&gL|=16lU}t7rc(BL&={$UKj-u+ZlOKN)G|}pq z@3;SujWJbjCA$wN1LUt2@y_*D$jPUzq2&u_fl}lkajj@zx`QB|5Xgr%)LeuY1xvw zg<+BJhSEBQabnQodOnl@;z{O;%d;dCH6#t%kLOJImiBV?8WyM+nRD5S^1JQw@ zV|OvszFm|$MN%q{vxg*9ck*h0ldQ$eaZad~t}A^x%O?-SK8wf$iHZ#Zh^|G><65W* z)^K#zTVTP_C1$2uI*SZG*$ukyWyKz;NV8adNsLGLSW!{-K}2r8_{0Z+2GjW&5jOas zB3E)!Kv_{xxzUPT^aKmHH*S6MJ);eI}*Pq;bz1_ zSwGGL3Ew^?Bz#+3JGb>BH>tfRh|#02(VNtt0$2ZH01Hkm!Z(JO`4ZRy! za5IHoz;m4MRMK7xy}0)~?LC7>b~#5ZrO`cSV}SC3cAcM1N~MjIC-CC4Bw$Kp>cHsU zi~uB#HF6)<8LYcJ;KJK80uAq%YVJAe+R%24TofGMpVi#kusOl)_i3-b|7)>S<(55G z#c$BZiv8ai%JmuE{}s+o_xV3x<=;=>?<8ZF!jDk)F3mD{!7|?+ygO)5cms-ny61rR z9lXCcbW0a>1<>|*3=Q)G80!XL5D?A+CN$=<0L^0n-W}ZABSkhRPz06^{!x>yD|-a) z9f>pl`F>(qGxPJwi6mPSdvq-5>WOYh<_Y5Sn3i1|a#yfuZZx&p(6rx&}X5vI>C-EqnIi^SguJe*5j+L6G#% z29#veGL~Omw6vLeJex|#K6<1X>bZ0bDBGGq&5{M)f}u`k&;TkgOk@p`u135$d3T`A zRG{dRtbvaobMyv2el!Jr2EPW;e0T5@=$OzUlW^2Fp@Fv7fY(I8gL~=$EM#i}49JV0 zT?PUSD?mZ{*yZKtpMQ>EDLVzJvf%-#z&TX(nF=)vyceB6z{ub=Ve-KR`(*0cyMuqd zI{@HNgfpm#P!nY23;_5e0Efx_9DW9e4m&;vMS|4*=+*m)bbN@Q{qgSLj{t(4I@1jc z;2d-=Lii8CkZ0Dr1KBD67aq+P{N0A8bwvI!XS!xW@CO(hx(fa?8Oi_jI{WD-aH9*n z0s>5NkhqbxDV`;YJcj^^vL$O0#wNh`p8##kiU|G+6j__TI}k(=qcl7^fBh@KB896X z0JtAn1N=-r85_E0K~2Pfj*mUA@iUo7BC4D|`F{eeC|hVIS-J`S3Jf?x=taRrP%~@| z;CcZ31egi0A<4t|Y8Ju_bqYmyeg-9}qoJ}WQvyuz5Vy4q)Lo{?uA1-%rwVz?HXxv} zc!Hj8Ovj48@H7cyK;D92OsY9j?2J7TuqbwtYyn{cg{f{^ZXM_A%tn2O%0G@U=o1Yt zK9W8>F7viEIYIw-jQJhqC7}s@P-G-j7_hFYvIRb>;1GfP40url8eBNCeux@5m>^7p zdJCzKh>%2x4l4vAR|=ACWeFA&S%E;dpur1FZA({q3yQ}Mr98NAp;8sG8?VStpGUx&Ko@DsWkB&5v~0};zm4oMkf4S} z8~z5~WBJ4wKP+dkjkV~9cLyc6x@8K#f%kqHu6oVSdGhBkK!k$ABiYe4#GCT5OAACy zM#m7pC2$c@`Jj7j=CPy`?i>Nkwcp5}zai=VjjVgeX!ZFGjNn)|Af_EL$s^viDu_;K zTZl|)iN1bx$K)%x_e3LT2+e}lO?=Ql21fWRXj?q)IndsMriBOeBN5l1QN;#a1S;8@ z2n|=YxC_cpKLJKpRbCU#kUEMeVD#@NP$Odp9@^j+q%>w|`o0eiD^eAfDSYrDcqLH? zShoOOQ;a%wR65BUXZsGf?}WM5^sEyETZO-Vkx>1A=- zt~21yf@0}l<_QZqj|j6sL=?Q2WJ{Jb-GJf|kTnY$Dimdh1T1~PM+~t_1iEd_Y%AQ* z8~YCguRm4(=fKN7{A{Km3s=@PHyh+`_kYFHv-tZD7yJCruk=s$?PKsSG*haw0*}y- zTWFfR3@X?dA3+l@ zMCb?-WDBBWBpn}spHB|pGw#+|R8T|Bv@ncE0^`wrdum>M48Zv8((0k{cga%|7-yA` zK;Rq`ILGtiv12Jd@P-5}%7;m|4?(;zJF=^TuP(R#>u~M94*v%J3jT#RQJ4($FHXu> zhKl$Zn3jQ-AiP!l3W{Gy^%MMG^wq8MWj%l+-h_{~p@6TT6b6b)I)>uI{Uc@6H)b}H z0&LO#Cg6JSv#Ym-2~_zH*uroc5L1F6kReog0Dlflod3#h@V89_uQLk%h6&tIxu%cCXfw;OEh9;dWcdRSQZ6G!ZOD7a^Lw*%Fmn zKbI_}w%$aF#s;s#g>FnAlNFs;-q?K2HQ-D)WlJ}f$=c2@53RqODOG+ZKThEidu6kTM}s>u)_3RzOae1gR~ySmX1H|BCv#R{7nCe^hMNng?62{A`uBpDWB ze4F4aI88Lm&Cg&JkbnGhqgj@JetnCXwa%RS#geJGaX=B|3X-Jtt>G}Z1C;mVclU%L@zdeiM6&E$P9R)5lyVO z{?N9d30ID6Ql+ecO{%}ixLRlxkB=+SiU##RJf2F;~O6}xz;(lL=+!D^!3-zo(=zH<9@sifulXl_}q8C|N7bA zJ8Zntr??hErQ5IdsSbot`4-bV)!h{;&o#j}_iDLV#zcb@th)|;FVTpwB3+B@z}b`l zDOr!Ka4QAeFW<_al<5Bv41S?N18thg&#k;bnf{+r`Sd)N|LOd^Sls)6zKj2d)ok_X zhYJLcOXPZ<99N9zMfd0W{v5YseE5!kwrs5pEp!XcOo%~U4Lec!=SS1xEojEQ zK`Q^;#znj6#4>ozl&$1rW|e<_@XL469aJE?&1F8$py3b*zJjw@uPmu{4-HJ}PS^<* zbREJeaS$NbfUn>@&SG5s&Z&J54T91xFyZl)x<@BAz`?{hhq{Pi;oM7XIW_O0;e>E) zg@M<*Wh@^B-Ny8PiUf0!ZY^WUpc>ouX~3DnFC%?k#!lT7e!06oXZp;ZJ%(m$u@G^w zp(+o%nzN#=KK^dY(CsXufJ9wwqSD=TWJ1eQV1&FDh{bml`Rym=OdNiqoVVor!k7BQD3P9Z**RCmBofJj+jV6k%j4_rfZKVMa3h! z&W)KH(NeftL)We>YliNLZ?mxAenmKj#-j*LeD_nHL!u{e=Mnvc>sI`#j+!vHXLYdq zDO@6V4etLU+P_z=PHJrs`I%`1Bn0wBt85j!mkcZtt{1KJw@r*DPMg_Q1&(=Ju}ntS z#2xOB(= zpW*5E$CoV-LcDn#^NKR!g=vz#wVc7KTbv)E4FoiS-7yJGsI!HLra}wJXhN;OZEV@6 z2xp)yk1fA}Fy_(-z}?K4{>CCAprnS|;@8g!6aCBPI3>+e8GU;c$;Y^e%E30jeoh+J z%{69fb$*ji1dPNs#2DoN(sD&t@cE zkP|m{CPwMEc*{0{>HH_Qb9BCwN#WGpw#Nr25e!Ov0!UJrM?8G7&7|emBp3wRZi@z) zjQI1T;Hs*#1nuC|bTxxJ0rU0lnBUk4^>#7maTY_oe_7YmCf)^azwPyK5#^&>W20_} zbUL~~eB={$f254Ew!JKbT#Q~PmoY{exRb4~B#mVarzCN@iSF^-(j9aJ4MR5`0|x`q zbAMD$bkq*a;eawr*6@J}+p75(sJN~VRPhzQd)P{9B6r~0>j1(w*hBVx*MyoYSB8{! zJ26o*6kzeYE5K521-Oxrrtmf84pu_Vt#4nY!c4e#l~6Y*$mCRUb4iY01PJzMrk-o& zY_~$nb3$25=3}>)GE@{WNl$UXcS}CUbPc+lwASjVTfE{ejIq;Fi&wT-dIj4spk_VJ z9PJ*lok;0DKKk7RJoaLl@=Jnggo@hoO91c&(=n$2#3pCXc3VuiYtU9t=XFuk~D(iapx%&w+TVd1R0J-PpRq)$w}}Z`hWri%%qYek%F47uGpm~ z(Se%4n-e@Ck`BN8k(fLwP`HaEqG3Ly9KVDsZGG~R1TNvw zC0uQb#Gor=D5+Ae6rfwk&?~0*ejRB_;z&x;&7Db8I=F=NnmdrDTZm3r(!?lKx@ZJ72kMos**Fj`}qbHQX-;S<1Uoh_D-?5UqnC)Wb9wHnxI)ZsT$ z012d~EDXC+2C2{zmWAyVLkp_<9Ins}NtIS+T8uZ@OhW|M3^QfXFx@fUaQY`PwdZI!4NZpM^%Cp5JQu*8F^jm)fBj_aMYBR;bDrsC$~%A)3}qZ%sB9@9|OjgC+-A#raQ`n=(c_l2ailf;kyrD5BAM3zI^ zy=?UA#s)Wll9?HkzfkEG(w6`yBzvLqw4G4My#6lxppgBa6brS3aKJ44zj#)-K(RiN z_dg03`}~jJIsYU3Y(Wj(3E5#b$ck;?;JqVIMM{BWl{<1(qxbin2KBM`7fRb@TD&^T z>U1%+cD)OUO^1B@4{!3gKjJ$NOnu8%pb4NR@H5j^aE?RNwShm6bpt?Nm^h^9V@Hb~ zg9@BMO+@-x*HCo?4Hq5Z#)57bP_TdrEnrS`TM@wsBBzvyR2CRZ2QSCs@U4twwEPA! ztT2EbA$5N0A$5Kl@b;%7MH(a_b$+@Fq&fjs*)FfnGSTW}Fw5QklK*)G4%S0a|i zFneblYX>-Hx*WU6M68{`F>_}eYX>-Hx*Yqo60tOnnLFcHRepw|%d)ZaM<54Iiv9{T zifb^ZaA(N5Y#Td&1T*JkV3)=@0%a#-8seSlGq0Gf{o9%BGk3_pc8GyyfP6#WksIHJB_m8GEmO40+Uk7aVPo(|-pmFBae{ zC}ykXP`em*E}UIpc+1a-9nH@WUfFuttsO&24P_(=~^;b~JR@dQ{F(f@m ziC8+7Y)yn?S%c!2Fz{bvMLDw4CT-4Quflmn+dKsq0TO=y%`5cX%7LAl7(yMxGk96E zOn~n&I{We8@OJDshZknQhy4`jLlQ}&j_N(Q|x!@U*WEu>Ta!2Yc^*NaxDl{ji!t zZ_EE+zhMW2z4t)u%7jRBL3DQFgXm;BAv#H3i1ylE2y-JZL_5O^(O%(&X#aR#2=i@w zAscQ> zGWOnx1h(wm5jj}&#&SOpyx(T_2ZH?GBe7-vhro)MkTl9K+$wEU6ig&>lwTY%Db{mK zkUWVSnG|V`i4Yp8CdCitns5_OzE$6Zi~k#UPWWW(y%Py+*}Erlu;^|1CtSSWq=Ujm ze(#~!fk~0(q6ijlYf_{;DT2j;gN_Z{6y}D0igu=>qP@~n(f$!#73N#_RhWMgXN9@< zR%G*K@2<$hqc;{m@unRX=H6qmqXUpkmqmL=K8yAWr$sy2Yti1oZDDTcw`gZNF4`+S z7wsR>bz#1B--Y=nabB2v??pCW_U?;3JbGjC6>r*sVeUN`yEHLUqbzph!)UK?VziUJ z82%GZ`OFJM{<@}x*GJ5(@Ow`(DZJ7j#hk#hvci;DTRc1}W1``eY#|mA7$K4H$^sT= z;QcIDAhOL(++b~E)9)s3Y6eh9G&KG4AI8jdUK-1mjFjiRZ(yE9DvYDEs{{Jj1tLs% zgBL7lnBaX!lkxN4`2S`)8nF+@Xhng(?#%+A@=GAf<1sWGVng-G+hsDbvM@;--q}>3 z7JL)SH3gxTx}4w2+N*F2@D|{KDr>T8t3M*^R9Sm;*cW{}9&;$2f&BApGLV1%VHwCj z69z_hA}Z!k`nC#c_*K5mS?Cb4@r-$N-wyO1Wb@|@#w=^{>=O0R9NAooU2;xia=d~X z{`nhF;|T2LBIQzZ789y$YVLWe+KOBRH33PUNA{2wQh`9|FS@bM!^z@$y6 zCCRN579!s17(iT|7e%z*06zFCEg%n+g3%JRsthkP7f;|$M*K;(MO3< z^E?m1Ug@T#@hZ$u`^0ULNp`VNK1Iv3Yz8C%h&ps!Tc$ljTjU@BSwq9mJK8~h!OzgB zysQD#nF-!=m4^A@cmq~?2(Nt5JB^zzgggx8X8Bpk-Gle@Q{-NrpMP-0DM%vdl6v!V zw-_0Tj&XzdNNgWq+J>RqnuwRa-)-bD85z7Vg_gMq%jDVWcgEA0CC~D)X@0s6qvoec z{K-$x{B-pT<>?g7Pjk|Az7|d1pdLE|9Rl|HV?IA${SJBtkIwV(sAI^vS#$*Wk*O=T z1wp690p4_#Ob{T^H|5Ju=Nk}h8#1E#Vk4SuL(WNZ3y$*(MwnJYU=Sdp74Vvf{R<-q z45;vyJcme;xEwp)j2Vvx9It+IK7%1GIL^ZpiY%QgnF3{N0*wt8AMk!Lk%dWY$w^&j zg7BOrM>haobdJz3*+TYMs`lQ5->&_z^4<3|eiHz>gwU47HAJw(Rl+hDgoRjqTHMI2 zu@M?8-;ht0FIyJ#5bGJGEhxd@?v!^;Ph=AUY$|#?dY4unRr~+jyO!oCZXldZS61kXw7u%tR% zM1Mk7<^)$M-;lvi!VoWky4O8~A2S4KckukLJF`41(gag(+IvSh2T0^OI>9L;=pso| zB^1Ia&2v_3OKb+g!p;+h(=@*@iL_7UC;I8GlhfTm(yS<3P%%jy9WM&Zm=;uIwm^nb z5nt}Lz?89PRF0xqD4hnlTgvMYn67pl1lMQr&lT^5}RJiD7N6PBG zl0vG5sgNfK!{%0}mTb`4{B!gB*ZZHFKUi`>b!7u&wpCz%OvwQAQ)|PsKBmkXA63-B z{$$5w4pFM3SEY1$o|pRPJHoz*A!Qpbx+Dx0qy&5=b5xR9Rl;4ZbTL`gnz<4O2F+6^ zX4BjXakKXBJSqAkq+FE&O7_YKpGz097XFTWt5ofwBf2GPc@)bWPfNT?H%K7bLc{y&Cbg{U)J*bCUH* z37QctG(K?36pzqZ@v4{}O6sNlV1q4hD& zi>sF2&U)S*WYbGH@ejn*bL>gDg>E3S=uzD3*UFkn&zLxLr*2|rL$MHa($ z=%R*UG5j)b*FY{%z2R~j9i8U+X-XzVne+U(I+^Uy#gU+M?#SbMU}09pIxR?mbw{$q3^(E5^lr(`;|x_XP`(zb1DK`u!?5*x$!! znK4`j{-)w}K!2-MBmB4NL)Oj#a20Cw0Gd@=PZz){)#wAXskD|(fU8rZ7tpr$_VSCU zUh#D|tICp4j(C!hiTv8|f$)efWVrM-Hj&PSk3J^+ERor=Ua8!aPicM;`UNR;-)!t& zEY&TTEnmGCSEJskR_}3r?poDT-7a5~7qX%#nQOnq&&>FG*I(Ii?3<@T2_V0ghCxAmcI{YQWI!y>i0cID zGw)4Dob-{-`eovn#G&Y6l`PWBDu*!z64;!leb9=jLez?{;^5 zZGYi}!d155XsebRE2DejT~m&OZcm@r+83>|4Q+<)$Vi$02qBsTS>_T+Yw;@nk;XY?9&Phnv5fznj0Czt#W#1HGK$8vxt_0O^q^X#fBK literal 0 HcmV?d00001