diff --git a/client/components.json b/client/components.json index ee4191d..47080d5 100644 --- a/client/components.json +++ b/client/components.json @@ -12,6 +12,9 @@ }, "aliases": { "components": "@/components", - "utils": "@/utils" + "utils": "@/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" } -} +} \ No newline at end of file diff --git a/client/src/components/app/dashboard.tsx b/client/src/components/app/dashboard.tsx index 4fd65b8..46a2680 100644 --- a/client/src/components/app/dashboard.tsx +++ b/client/src/components/app/dashboard.tsx @@ -8,6 +8,7 @@ import { } from "@/components/ui/dropdown-menu"; import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet"; import { + BriefcaseBusiness, CircleUser, Earth, LibraryBig, @@ -20,11 +21,25 @@ import { Link, useLocation } from "wouter"; import MenuLink from "../ui/menu-link"; import { Toaster } from "../ui/toaster"; import { TooltipProvider } from "../ui/tooltip"; -import Routes from "./routes"; +import { Routes } from "./routes"; +import { OrgSelector } from "../ui/org-selector"; +import { useContext, useState } from "react"; +import { UserContext } from "@/userContext"; export function Dashboard() { const [location] = useLocation(); + const [isOrgDropDownOpen, setOrgDropDownOpen] = useState(false) + const { currentOrganization, userOrgsQuery } = useContext(UserContext); + + const orgSelectorItems = userOrgsQuery?.data?.map(organization => { + return { + key: organization.orgs.id, + linkProps: { to: `~/org/${organization.orgs?.id}` }, + text: organization.orgs.name, + } + }); + return (
@@ -110,6 +125,33 @@ export function Dashboard() {
+ { orgSelectorItems?.length! > 1 && ( + setOrgDropDownOpen(state)} + > + + + + + setOrgDropDownOpen(false)} + /> + + + )} + )) + } +
+ ) +} \ No newline at end of file diff --git a/client/src/main.tsx b/client/src/main.tsx index 6a5180c..e25f831 100644 --- a/client/src/main.tsx +++ b/client/src/main.tsx @@ -1,13 +1,15 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { httpBatchLink } from "@trpc/client"; -import React, { useEffect, useState } from "react"; +import React from "react"; import ReactDOM from "react-dom/client"; -import { Dashboard } from "./components/app/dashboard"; +import { Scaffold } from "./components/app/routes"; import Unauthenticated from "./components/app/unauthenticated"; import { API_URL } from "./constants"; import "./main.css"; -import UserContext from "./userContext"; +import { UserProvider } from "./userContext"; import { trpc } from "./utils"; +import { Dashboard } from "./components/app/dashboard"; +import { WithUser } from "./components/app/withUser"; const queryClient = new QueryClient(); const trpcClient = trpc.createClient({ @@ -25,39 +27,20 @@ const trpcClient = trpc.createClient({ }); export function App() { - const [user, setUser] = useState(); - const [isLoading, setIsLoading] = useState(true); - - useEffect(() => { - setIsLoading(true); - fetch(`${API_URL}/me`, { credentials: "include" }) - .then((r) => { - if (r.status != 200) { - throw new Error("Not authorized"); - } - return r.json(); - }) - .then(setUser) - .then(() => setIsLoading(false)) - .catch(() => { - setIsLoading(false); - }); - }, []); - - if (isLoading) { - return null; - } - return ( - - - - {user ? : } - - - - + + + + } + Else={} + /> + + + + + ); } diff --git a/client/src/userContext.tsx b/client/src/userContext.tsx index 0f6eea8..44b5085 100644 --- a/client/src/userContext.tsx +++ b/client/src/userContext.tsx @@ -1,8 +1,63 @@ -import { createContext } from "react"; +import { + createContext, + FC, + PropsWithChildren, + useMemo, + useState +} from "react"; +import { UseTRPCQueryResult } from "@trpc/react-query/shared"; +import { TRPCClientErrorLike } from '@trpc/react-query'; +import { trpc, AppRouter, RouterOutput } from "./utils"; -const UserContext = createContext<{ +type UserOrgs = RouterOutput['orgs']['ofCurrentUser']; +type UserOrgsQuery = UseTRPCQueryResult>; + +export interface UserContextProps { user: any; + userOrgsQuery?: UserOrgsQuery, setUser: React.Dispatch; -}>({ user: null, setUser: () => {} }); + currentOrganizationId?: number; + setCurrentOrganizationId: React.Dispatch; + currentOrganization?: UserOrgs[number]; +} + +export const UserContext = createContext({ + user: null, + setUser: () => { }, + setCurrentOrganizationId: () => { } +}); + +export const UserProvider: FC = ({ children }) => { + const [user, setUser] = useState(); + const [currentOrganizationId, setCurrentOrganizationId] = useState(); + const userOrgsQuery = trpc.orgs.ofCurrentUser.useQuery(undefined, { + enabled: !!user?.id, + }); + const currentOrganization = userOrgsQuery?.data?.find(org => org.orgs.id === currentOrganizationId); -export default UserContext; + const value = useMemo(() => ({ + user, + setUser: (user: any) => { + console.trace(`setting user`); + setUser(user); + }, + currentOrganization, + currentOrganizationId, + setCurrentOrganizationId: (id: any) => { + console.trace(`set ID state`); + setCurrentOrganizationId(id); + }, + userOrgsQuery, + }), + [ + user?.id, + currentOrganizationId, + userOrgsQuery?.status, + ] + ); + return ( + + {children} + + ); +} \ No newline at end of file diff --git a/client/tsconfig.json b/client/tsconfig.json index 5eb5e91..c2cba48 100644 --- a/client/tsconfig.json +++ b/client/tsconfig.json @@ -16,6 +16,7 @@ /* Linting */ "strict": true, + "strictNullChecks": true, "noUnusedLocals": true, "noUnusedParameters": true, "noFallthroughCasesInSwitch": true, diff --git a/common/types.ts b/common/types.ts index 9189acd..c5f194b 100644 --- a/common/types.ts +++ b/common/types.ts @@ -117,3 +117,5 @@ export interface CompletionStats { steps: StepCompletionStats[]; generatedAt: string; } + +export type { Organizations } from '../server/src/data/types'; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d3df73d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3.8' + +services: + + ctdl-xtra: + build: . + container_name: ctdl-xtra + depends_on: + - postgres + - redis + environment: + DATABASE_URL: postgres://xtra:xtra@postgres:5432/cdtlxtra + REDIS_URL: redis://redis:6379 + ports: + - "3000:3000" + volumes: + - .:/app + ctdl-xtra-worker: + build: + context: . + dockerfile: worker.Dockerfile + container_name: ctdl-xtra-worker + depends_on: + - postgres + - redis + environment: + DATABASE_URL: postgres://xtra:xtra@postgres:5432/xtra + REDIS_URL: redis://redis:6379 + volumes: + - .:/app + + postgres: + image: postgres:latest + container_name: my_postgres + restart: always + environment: + POSTGRES_USER: xtra + POSTGRES_PASSWORD: xtra + POSTGRES_DB: xtra + ports: + - "5432:5432" + volumes: + - pg_data:/var/lib/postgresql/data + + redis: + image: redis:latest + container_name: my_redis + restart: always + ports: + - "6379:6379" + volumes: + - redis_data:/data + +volumes: + pg_data: + redis_data: diff --git a/server/.env.example b/server/.env.example index 147bcb6..3c179d8 100644 --- a/server/.env.example +++ b/server/.env.example @@ -1,8 +1,12 @@ -DATABASE_URL=postgresql://... # Point to your local PG installation +DATABASE_URL=postgresql://postgres:pass@localhost:5432/postgres # Point to your local PG installation +# format: postgresql://:@:/ + EXTRACTION_FILES_PATH=/path/to/local/folder # Create a local folder to store these -ENCRYPTION_KEY= # Generate a random long string (UUID) -COOKIE_KEY= # Generate a random long string (UUID) -CLIENT_PATH=/path/to/local/client/dist/folder # In the client project there will be a dist folder when you compile, add that here +ENCRYPTION_KEY= # Generate a random 32 character string (UUID) +COOKIE_KEY= # Generate a hex key with `pnpm exec secure-session | xxd -p -c 0` +CLIENT_PATH= # In the client project there will be a dist folder when you compile, add that here +# you can run `realpath ../client/dist` after client build to get the exact value needed + FRONTEND_URL=http://localhost:5173 #SMTP_USER= You can leave blank for development #SMTP_PASSWORD= You can leave blank for development diff --git a/server/README.md b/server/README.md index c372954..5822fb9 100644 --- a/server/README.md +++ b/server/README.md @@ -12,16 +12,21 @@ for the application. - **Web scraping**: Puppeteer - **Email**: React Email for templating and Nodemailer for sending - **Database**: PostgreSQL with Drizzle ORM +- **Cache**: Redis - **Error monitoring**: Airbrake ## Setup -1. Install [node.js](https://nodejs.org/en) (20+) and [pnpm](https://pnpm.io/) +1. Install [node.js](https://nodejs.org/en) (20+) (using [nvm](https://github.com/nvm-sh/nvm) is a plus) and [pnpm](https://pnpm.io/) 2. Install dependencies: ```bash pnpm install +pnpm dlx puppeteer browsers install + +# or specific version +pnpm dlx puppeteer browsers install chrome@130.0.6723.58 ``` 3. Set up environment variables: @@ -30,6 +35,12 @@ pnpm install cp .env.example .env # edit your env vars in .env ``` +Generate keys required for [secure-session](https://github.com/fastify/fastify-secure-session) encryption (32 bytes - 64 hex encoded characters) +```bash +pnpm exec secure-session | xxd -p -c 0 +``` +Use generated key in the `COOKIE_KEY` env variable. + ## Development Run the development server: @@ -50,6 +61,12 @@ Run the email preview server: pnpm run dev:email ``` +Docker: +```bash +docker run --env=POSTGRES_PASSWORD=pass -p 5432:5432 -d postgres:13-alpine +docker run -p 6379:6379 -d redis +``` + ## Database The application uses PostgreSQL with Drizzle ORM for database management. @@ -66,6 +83,8 @@ Apply migrations: pnpm run db:migrate ``` +To seed a development user, use `src/createUser.ts`: + ## Testing Run tests: diff --git a/server/migrations/0002_cooing_gauntlet.sql b/server/migrations/0002_cooing_gauntlet.sql new file mode 100644 index 0000000..61c66e0 --- /dev/null +++ b/server/migrations/0002_cooing_gauntlet.sql @@ -0,0 +1,21 @@ +CREATE TYPE "public"."role_type" AS ENUM('viewer', 'member', 'admin');--> statement-breakpoint +CREATE TABLE "memberships" ( + "org_id" integer NOT NULL, + "user_id" integer NOT NULL, + "role" "role_type" DEFAULT 'member' +); +--> statement-breakpoint +CREATE TABLE "orgs" ( + "id" serial PRIMARY KEY NOT NULL, + "name" text DEFAULT 'null', + "description" text DEFAULT 'null', + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "catalogues" ADD COLUMN "org_id" integer NOT NULL;--> statement-breakpoint +ALTER TABLE "settings" ADD COLUMN "org_id" integer NOT NULL;--> statement-breakpoint +ALTER TABLE "users" ADD COLUMN "is_staff" boolean DEFAULT false NOT NULL;--> statement-breakpoint +ALTER TABLE "memberships" ADD CONSTRAINT "memberships_org_id_orgs_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."orgs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "memberships" ADD CONSTRAINT "memberships_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "catalogues" ADD CONSTRAINT "catalogues_org_id_orgs_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."orgs"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "settings" ADD CONSTRAINT "settings_org_id_orgs_id_fk" FOREIGN KEY ("org_id") REFERENCES "public"."orgs"("id") ON DELETE no action ON UPDATE no action; \ No newline at end of file diff --git a/server/migrations/meta/0002_snapshot.json b/server/migrations/meta/0002_snapshot.json new file mode 100644 index 0000000..4dc0acd --- /dev/null +++ b/server/migrations/meta/0002_snapshot.json @@ -0,0 +1,1255 @@ +{ + "id": "7d940431-8c0e-451e-b69d-ac5a1826cb84", + "prevId": "a30ef025-2510-4069-95cd-7e87c7728d5d", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.catalogues": { + "name": "catalogues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "thumbnail_url": { + "name": "thumbnail_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "catalogues_org_id_orgs_id_fk": { + "name": "catalogues_org_id_orgs_id_fk", + "tableFrom": "catalogues", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "catalogues_url_unique": { + "name": "catalogues_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.crawl_pages": { + "name": "crawl_pages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "crawl_step_id": { + "name": "crawl_step_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "page_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "screenshot": { + "name": "screenshot", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fetch_failure_reason": { + "name": "fetch_failure_reason", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "data_type": { + "name": "data_type", + "type": "page_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "data_extraction_started_at": { + "name": "data_extraction_started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "crawl_pages_extraction_idx": { + "name": "crawl_pages_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_status_idx": { + "name": "crawl_pages_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_data_type_idx": { + "name": "crawl_pages_data_type_idx", + "columns": [ + { + "expression": "data_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_pages_step_idx": { + "name": "crawl_pages_step_idx", + "columns": [ + { + "expression": "crawl_step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "crawl_pages_extraction_id_extractions_id_fk": { + "name": "crawl_pages_extraction_id_extractions_id_fk", + "tableFrom": "crawl_pages", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "crawl_pages_crawl_step_id_crawl_steps_id_fk": { + "name": "crawl_pages_crawl_step_id_crawl_steps_id_fk", + "tableFrom": "crawl_pages", + "tableTo": "crawl_steps", + "columnsFrom": [ + "crawl_step_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "crawl_pages_extraction_id_url_unique": { + "name": "crawl_pages_extraction_id_url_unique", + "nullsNotDistinct": false, + "columns": [ + "extraction_id", + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.crawl_steps": { + "name": "crawl_steps", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "step": { + "name": "step", + "type": "step", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "parent_step_id": { + "name": "parent_step_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "crawl_steps_extraction_idx": { + "name": "crawl_steps_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "crawl_steps_parent_step_idx": { + "name": "crawl_steps_parent_step_idx", + "columns": [ + { + "expression": "parent_step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "crawl_steps_extraction_id_extractions_id_fk": { + "name": "crawl_steps_extraction_id_extractions_id_fk", + "tableFrom": "crawl_steps", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "crawl_steps_parent_step_id_crawl_steps_id_fk": { + "name": "crawl_steps_parent_step_id_crawl_steps_id_fk", + "tableFrom": "crawl_steps", + "tableTo": "crawl_steps", + "columnsFrom": [ + "parent_step_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_items": { + "name": "data_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "dataset_id": { + "name": "dataset_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "crawl_page_id": { + "name": "crawl_page_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "structured_data": { + "name": "structured_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "text_inclusion": { + "name": "text_inclusion", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_items_dataset_idx": { + "name": "data_items_dataset_idx", + "columns": [ + { + "expression": "dataset_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_items_crawl_page_idx": { + "name": "data_items_crawl_page_idx", + "columns": [ + { + "expression": "crawl_page_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_items_dataset_id_datasets_id_fk": { + "name": "data_items_dataset_id_datasets_id_fk", + "tableFrom": "data_items", + "tableTo": "datasets", + "columnsFrom": [ + "dataset_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_items_crawl_page_id_crawl_pages_id_fk": { + "name": "data_items_crawl_page_id_crawl_pages_id_fk", + "tableFrom": "data_items", + "tableTo": "crawl_pages", + "columnsFrom": [ + "crawl_page_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.datasets": { + "name": "datasets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "catalogue_id": { + "name": "catalogue_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "datasets_catalogue_idx": { + "name": "datasets_catalogue_idx", + "columns": [ + { + "expression": "catalogue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_extraction_idx": { + "name": "datasets_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "datasets_catalogue_id_catalogues_id_fk": { + "name": "datasets_catalogue_id_catalogues_id_fk", + "tableFrom": "datasets", + "tableTo": "catalogues", + "columnsFrom": [ + "catalogue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "datasets_extraction_id_extractions_id_fk": { + "name": "datasets_extraction_id_extractions_id_fk", + "tableFrom": "datasets", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "datasets_catalogue_id_extraction_id_unique": { + "name": "datasets_catalogue_id_extraction_id_unique", + "nullsNotDistinct": false, + "columns": [ + "catalogue_id", + "extraction_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.extraction_logs": { + "name": "extraction_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "log": { + "name": "log", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "log_level": { + "name": "log_level", + "type": "log_level", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'INFO'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "extraction_logs_extraction_idx": { + "name": "extraction_logs_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "extraction_logs_extraction_id_extractions_id_fk": { + "name": "extraction_logs_extraction_id_extractions_id_fk", + "tableFrom": "extraction_logs", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.extractions": { + "name": "extractions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "recipe_id": { + "name": "recipe_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "completion_stats": { + "name": "completion_stats", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "extraction_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "extractions_recipe_idx": { + "name": "extractions_recipe_idx", + "columns": [ + { + "expression": "recipe_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "extractions_recipe_id_recipes_id_fk": { + "name": "extractions_recipe_id_recipes_id_fk", + "tableFrom": "extractions", + "tableTo": "recipes", + "columnsFrom": [ + "recipe_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "role_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false, + "default": "'member'" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_org_id_orgs_id_fk": { + "name": "memberships_org_id_orgs_id_fk", + "tableFrom": "memberships", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "memberships_user_id_users_id_fk": { + "name": "memberships_user_id_users_id_fk", + "tableFrom": "memberships", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_api_calls": { + "name": "model_api_calls", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "extraction_id": { + "name": "extraction_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "provider_model", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "call_site": { + "name": "call_site", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_token_count": { + "name": "input_token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "output_token_count": { + "name": "output_token_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "model_api_calls_extraction_idx": { + "name": "model_api_calls_extraction_idx", + "columns": [ + { + "expression": "extraction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "model_api_calls_extraction_id_extractions_id_fk": { + "name": "model_api_calls_extraction_id_extractions_id_fk", + "tableFrom": "model_api_calls", + "tableTo": "extractions", + "columnsFrom": [ + "extraction_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.orgs": { + "name": "orgs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'null'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'null'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.recipes": { + "name": "recipes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "catalogue_id": { + "name": "catalogue_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "detection_failure_reason": { + "name": "detection_failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "recipe_detection_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'WAITING'" + } + }, + "indexes": { + "recipes_catalogue_idx": { + "name": "recipes_catalogue_idx", + "columns": [ + { + "expression": "catalogue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "recipes_catalogue_id_catalogues_id_fk": { + "name": "recipes_catalogue_id_catalogues_id_fk", + "tableFrom": "recipes", + "tableTo": "catalogues", + "columnsFrom": [ + "catalogue_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.settings": { + "name": "settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_encrypted": { + "name": "is_encrypted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "encrypted_preview": { + "name": "encrypted_preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "org_id": { + "name": "org_id", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "settings_org_id_orgs_id_fk": { + "name": "settings_org_id_orgs_id_fk", + "tableFrom": "settings", + "tableTo": "orgs", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "settings_key_unique": { + "name": "settings_key_unique", + "nullsNotDistinct": false, + "columns": [ + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_staff": { + "name": "is_staff", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.extraction_status": { + "name": "extraction_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "COMPLETE", + "STALE", + "CANCELLED" + ] + }, + "public.log_level": { + "name": "log_level", + "schema": "public", + "values": [ + "INFO", + "ERROR" + ] + }, + "public.page_status": { + "name": "page_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "SUCCESS", + "ERROR" + ] + }, + "public.page_type": { + "name": "page_type", + "schema": "public", + "values": [ + "COURSE_DETAIL_PAGE", + "CATEGORY_LINKS_PAGE", + "COURSE_LINKS_PAGE" + ] + }, + "public.provider": { + "name": "provider", + "schema": "public", + "values": [ + "openai" + ] + }, + "public.provider_model": { + "name": "provider_model", + "schema": "public", + "values": [ + "gpt-4o" + ] + }, + "public.recipe_detection_status": { + "name": "recipe_detection_status", + "schema": "public", + "values": [ + "WAITING", + "IN_PROGRESS", + "SUCCESS", + "ERROR" + ] + }, + "public.role_type": { + "name": "role_type", + "schema": "public", + "values": [ + "viewer", + "member", + "admin" + ] + }, + "public.step": { + "name": "step", + "schema": "public", + "values": [ + "FETCH_ROOT", + "FETCH_PAGINATED", + "FETCH_LINKS" + ] + }, + "public.url_pattern_type": { + "name": "url_pattern_type", + "schema": "public", + "values": [ + "page_num", + "offset" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/server/migrations/meta/_journal.json b/server/migrations/meta/_journal.json index a879964..5b3480c 100644 --- a/server/migrations/meta/_journal.json +++ b/server/migrations/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1727814665852, "tag": "0001_normal_beast", "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1739203036407, + "tag": "0002_cooing_gauntlet", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/package.json b/server/package.json index 2e3dbcd..ee60eac 100644 --- a/server/package.json +++ b/server/package.json @@ -21,7 +21,8 @@ "bullmq": "^5.10.0", "cheerio": "1.0.0-rc.12", "dotenv": "^16.4.5", - "drizzle-orm": "^0.31.2", + "drizzle-orm": "^0.39.2", + "drizzle-seed": "^0.3.1", "fast-csv": "^5.0.1", "fast-equals": "^5.0.1", "fastify": "^4.26.2", @@ -47,7 +48,7 @@ "@types/react": "^18.3.3", "@types/turndown": "^5.0.4", "@vitest/ui": "^2.1.4", - "drizzle-kit": "^0.22.7", + "drizzle-kit": "^0.30.4", "nodemon": "^3.1.0", "react": "^18.3.1", "react-dom": "^18.3.1", diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 0526c67..79b86f5 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -42,8 +42,11 @@ importers: specifier: ^16.4.5 version: 16.4.5 drizzle-orm: - specifier: ^0.31.2 - version: 0.31.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1) + specifier: ^0.39.2 + version: 0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1) + drizzle-seed: + specifier: ^0.3.1 + version: 0.3.1(drizzle-orm@0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1)) fast-csv: specifier: ^5.0.1 version: 5.0.1 @@ -115,8 +118,8 @@ importers: specifier: ^2.1.4 version: 2.1.4(vitest@2.1.4) drizzle-kit: - specifier: ^0.22.7 - version: 0.22.7 + specifier: ^0.30.4 + version: 0.30.4 nodemon: specifier: ^3.1.0 version: 3.1.0 @@ -128,7 +131,7 @@ importers: version: 18.3.1(react@18.3.1) react-email: specifier: ^2.1.6 - version: 2.1.6(@swc/helpers@0.5.2)(eslint@9.7.0)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) + version: 2.1.6(@swc/helpers@0.5.2)(eslint@9.7.0)(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)) tsx: specifier: ^4.16.2 version: 4.16.2 @@ -268,6 +271,9 @@ packages: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} + '@drizzle-team/brocli@0.10.2': + resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} + '@emotion/is-prop-valid@0.8.8': resolution: {integrity: sha512-u5WtneEAr5IDG2Wv65yhunPSMLIpuKsbuOktRojfrEiEvRyC85LgPMZI63cr7NUqT8ZIGdSVg8ZKGxIug4lXcA==} @@ -276,9 +282,11 @@ packages: '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} + deprecated: 'Merged into tsx: https://tsx.is' '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} + deprecated: 'Merged into tsx: https://tsx.is' '@esbuild/aix-ppc64@0.19.11': resolution: {integrity: sha512-FnzU0LyE3ySQk7UntJO4+qIiQgI7KoODnZg5xzXIrFJlKd2P2gwHsHY4927xj9y5PJmJSzULiUCWmv7iWnNa7g==} @@ -2428,21 +2436,23 @@ packages: resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} engines: {node: '>=12'} - drizzle-kit@0.22.7: - resolution: {integrity: sha512-9THPCb2l1GPt7wxhws9LvTR0YG565ZlVgTuqGMwjs590Kch1pXu4GyjEArVijSF5m0OBj3qgdeKmuJXhKXgWFw==} + drizzle-kit@0.30.4: + resolution: {integrity: sha512-B2oJN5UkvwwNHscPWXDG5KqAixu7AUzZ3qbe++KU9SsQ+cZWR4DXEPYcvWplyFAno0dhRJECNEhNxiDmFaPGyQ==} hasBin: true - drizzle-orm@0.31.2: - resolution: {integrity: sha512-QnenevbnnAzmbNzQwbhklvIYrDE8YER8K7kSrAWQSV1YvFCdSQPzj+jzqRdTSsV2cDqSpQ0NXGyL1G9I43LDLg==} + drizzle-orm@0.39.2: + resolution: {integrity: sha512-cuopo+udkKEGGpSxCML9ZRQ43R01zYCTsbqCrb9kJkabx1QEwFlPIFoQfx6E6tuawtiX930Gwyrkwj4inlpzDg==} peerDependencies: '@aws-sdk/client-rds-data': '>=3' - '@cloudflare/workers-types': '>=3' - '@electric-sql/pglite': '>=0.1.1' - '@libsql/client': '*' - '@neondatabase/serverless': '>=0.1' + '@cloudflare/workers-types': '>=4' + '@electric-sql/pglite': '>=0.2.0' + '@libsql/client': '>=0.10.0' + '@libsql/client-wasm': '>=0.10.0' + '@neondatabase/serverless': '>=0.10.0' '@op-engineering/op-sqlite': '>=2' '@opentelemetry/api': ^1.4.1 '@planetscale/database': '>=1' + '@prisma/client': '*' '@tidbcloud/serverless': '*' '@types/better-sqlite3': '*' '@types/pg': '*' @@ -2452,12 +2462,13 @@ packages: '@xata.io/client': '*' better-sqlite3: '>=7' bun-types: '*' - expo-sqlite: '>=13.2.0' + expo-sqlite: '>=14.0.0' knex: '*' kysely: '*' mysql2: '>=2' pg: '>=8' postgres: '>=3' + prisma: '*' react: '>=18' sql.js: '>=1' sqlite3: '>=5' @@ -2470,6 +2481,8 @@ packages: optional: true '@libsql/client': optional: true + '@libsql/client-wasm': + optional: true '@neondatabase/serverless': optional: true '@op-engineering/op-sqlite': @@ -2478,6 +2491,8 @@ packages: optional: true '@planetscale/database': optional: true + '@prisma/client': + optional: true '@tidbcloud/serverless': optional: true '@types/better-sqlite3': @@ -2508,6 +2523,8 @@ packages: optional: true postgres: optional: true + prisma: + optional: true react: optional: true sql.js: @@ -2515,6 +2532,14 @@ packages: sqlite3: optional: true + drizzle-seed@0.3.1: + resolution: {integrity: sha512-F/0lgvfOAsqlYoHM/QAGut4xXIOXoE5VoAdv2FIl7DpGYVXlAzKuJO+IphkKUFK3Dz+rFlOsQLnMNrvoQ0cx7g==} + peerDependencies: + drizzle-orm: '>=0.36.4' + peerDependenciesMeta: + drizzle-orm: + optional: true + eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} @@ -3870,6 +3895,9 @@ packages: deprecated: < 22.8.2 is no longer supported hasBin: true + pure-rand@6.1.0: + resolution: {integrity: sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==} + queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -4879,6 +4907,8 @@ snapshots: '@jridgewell/trace-mapping': 0.3.9 optional: true + '@drizzle-team/brocli@0.10.2': {} + '@emotion/is-prop-valid@0.8.8': dependencies: '@emotion/memoize': 0.7.4 @@ -6074,7 +6104,7 @@ snapshots: dependencies: '@types/node': 20.12.7 tapable: 2.2.1 - webpack: 5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11) + webpack: 5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.12) transitivePeerDependencies: - '@swc/core' - esbuild @@ -6752,15 +6782,16 @@ snapshots: dotenv@16.4.5: {} - drizzle-kit@0.22.7: + drizzle-kit@0.30.4: dependencies: + '@drizzle-team/brocli': 0.10.2 '@esbuild-kit/esm-loader': 2.6.5 esbuild: 0.19.12 esbuild-register: 3.5.0(esbuild@0.19.12) transitivePeerDependencies: - supports-color - drizzle-orm@0.31.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1): + drizzle-orm@0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1): optionalDependencies: '@types/better-sqlite3': 7.6.9 '@types/pg': 8.11.9 @@ -6769,6 +6800,12 @@ snapshots: pg: 8.12.0 react: 18.3.1 + drizzle-seed@0.3.1(drizzle-orm@0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1)): + dependencies: + pure-rand: 6.1.0 + optionalDependencies: + drizzle-orm: 0.39.2(@types/better-sqlite3@7.6.9)(@types/pg@8.11.9)(@types/react@18.3.3)(better-sqlite3@9.5.0)(pg@8.12.0)(react@18.3.1) + eastasianwidth@0.2.0: {} editorconfig@1.0.4: @@ -6840,7 +6877,7 @@ snapshots: esbuild-register@3.5.0(esbuild@0.19.12): dependencies: - debug: 4.3.5 + debug: 4.3.7 esbuild: 0.19.12 transitivePeerDependencies: - supports-color @@ -8044,13 +8081,13 @@ snapshots: camelcase-css: 2.0.1 postcss: 8.4.38 - postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)): + postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)): dependencies: lilconfig: 3.1.2 yaml: 2.5.0 optionalDependencies: postcss: 8.4.38 - ts-node: 10.9.2(@types/node@20.12.7)(typescript@5.4.5) + ts-node: 10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5) postcss-nested@6.2.0(postcss@8.4.38): dependencies: @@ -8252,6 +8289,8 @@ snapshots: - typescript - utf-8-validate + pure-rand@6.1.0: {} + queue-microtask@1.2.3: {} queue-tick@1.0.1: {} @@ -8275,7 +8314,7 @@ snapshots: react: 18.3.1 scheduler: 0.23.2 - react-email@2.1.6(@swc/helpers@0.5.2)(eslint@9.7.0)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)): + react-email@2.1.6(@swc/helpers@0.5.2)(eslint@9.7.0)(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)): dependencies: '@babel/core': 7.24.5 '@babel/parser': 7.24.5 @@ -8315,7 +8354,7 @@ snapshots: source-map-js: 1.0.2 stacktrace-parser: 0.1.10 tailwind-merge: 2.2.0 - tailwindcss: 3.4.0(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) + tailwindcss: 3.4.0(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)) typescript: 5.1.6 transitivePeerDependencies: - '@opentelemetry/api' @@ -8731,7 +8770,7 @@ snapshots: dependencies: '@babel/runtime': 7.24.8 - tailwindcss@3.4.0(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)): + tailwindcss@3.4.0(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)): dependencies: '@alloc/quick-lru': 5.2.0 arg: 5.0.2 @@ -8750,7 +8789,7 @@ snapshots: postcss: 8.4.38 postcss-import: 15.1.0(postcss@8.4.38) postcss-js: 4.0.1(postcss@8.4.38) - postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) + postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5)) postcss-nested: 6.2.0(postcss@8.4.38) postcss-selector-parser: 6.1.1 resolve: 1.22.8 @@ -8801,14 +8840,14 @@ snapshots: dependencies: bintrees: 1.0.2 - terser-webpack-plugin@5.3.10(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)(webpack@5.93.0(esbuild@0.19.12)): + terser-webpack-plugin@5.3.10(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)(webpack@5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)): dependencies: '@jridgewell/trace-mapping': 0.3.25 jest-worker: 27.5.1 schema-utils: 3.3.0 serialize-javascript: 6.0.2 terser: 5.31.3 - webpack: 5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11) + webpack: 5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.12) optionalDependencies: '@swc/core': 1.3.101(@swc/helpers@0.5.2) esbuild: 0.19.11 @@ -8871,7 +8910,7 @@ snapshots: ts-interface-checker@0.1.13: {} - ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5): + ts-node@10.9.2(@swc/core@1.3.101(@swc/helpers@0.5.2))(@types/node@20.12.7)(typescript@5.4.5): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.11 @@ -8888,6 +8927,8 @@ snapshots: typescript: 5.4.5 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 + optionalDependencies: + '@swc/core': 1.3.101(@swc/helpers@0.5.2) optional: true tslib@2.6.2: {} @@ -9046,7 +9087,7 @@ snapshots: webpack-sources@3.2.3: {} - webpack@5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11): + webpack@5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.12): dependencies: '@types/eslint-scope': 3.7.7 '@types/estree': 1.0.5 @@ -9069,7 +9110,7 @@ snapshots: neo-async: 2.6.2 schema-utils: 3.3.0 tapable: 2.2.1 - terser-webpack-plugin: 5.3.10(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)(webpack@5.93.0(esbuild@0.19.12)) + terser-webpack-plugin: 5.3.10(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)(webpack@5.93.0(@swc/core@1.3.101(@swc/helpers@0.5.2))(esbuild@0.19.11)) watchpack: 2.4.1 webpack-sources: 3.2.3 transitivePeerDependencies: diff --git a/server/src/appRouter.ts b/server/src/appRouter.ts index 7dc14f4..4d81bed 100644 --- a/server/src/appRouter.ts +++ b/server/src/appRouter.ts @@ -10,6 +10,7 @@ import { extractionsRouter } from "./routers/extractions"; import { recipesRouter } from "./routers/recipes"; import { settingsRouter } from "./routers/settings"; import { usersRouter } from "./routers/users"; +import { orgsRouter } from "./routers/orgs"; import { DetectConfigurationProgress } from "./workers"; const appRouter = router({ @@ -19,6 +20,7 @@ const appRouter = router({ recipes: recipesRouter, extractions: extractionsRouter, users: usersRouter, + orgs: orgsRouter, }); export { appRouter }; diff --git a/server/src/data/catalogues.ts b/server/src/data/catalogues.ts index 33e3878..f125353 100644 --- a/server/src/data/catalogues.ts +++ b/server/src/data/catalogues.ts @@ -1,4 +1,4 @@ -import { desc, eq, sql } from "drizzle-orm"; +import { desc, eq, sql, and, inArray, } from "drizzle-orm"; import db from "../data"; import { catalogues, extractions, recipes } from "../data/schema"; @@ -34,9 +34,12 @@ export async function findCatalogueById(id: number) { return result; } -export async function findCatalogueByUrl(url: string) { +export async function findCatalogueByUrl(url: string, orgIds: number[]) { return db.query.catalogues.findFirst({ - where: (catalogues, { eq }) => eq(catalogues.url, url), + where: and( + eq(catalogues.url, url), + inArray(catalogues.orgId, orgIds), + ), }); } @@ -51,9 +54,10 @@ export async function findLatestExtractionsForCatalogue(catalogueId: number) { return catExtractions.map((e) => e.extractions); } -export async function findCatalogues(limit: number = 20, offset?: number) { +export async function findCatalogues(orgId: number, limit: number = 20, offset?: number) { offset = offset || 0; return db.query.catalogues.findMany({ + where: eq(catalogues.orgId, orgId), limit, offset, with: { @@ -65,11 +69,12 @@ export async function findCatalogues(limit: number = 20, offset?: number) { export async function createCatalogue( name: string, url: string, - thumbnailUrl?: string + orgId: number, + thumbnailUrl?: string, ) { const result = await db .insert(catalogues) - .values({ name, url, thumbnailUrl }) + .values({ name, url, thumbnailUrl, orgId }) .returning(); return result[0]; } diff --git a/server/src/data/orgs.ts b/server/src/data/orgs.ts new file mode 100644 index 0000000..760c2c9 --- /dev/null +++ b/server/src/data/orgs.ts @@ -0,0 +1,10 @@ +import { eq } from 'drizzle-orm' + +import db from "../data"; +import { memberships, orgs } from "./schema"; + +export async function findOrgsByUser(userId: number) { + return db.selectDistinct() + .from(orgs) + .innerJoin(memberships, eq(memberships.userId, userId)) +} \ No newline at end of file diff --git a/server/src/data/schema.ts b/server/src/data/schema.ts index cf78be6..0c6fd8f 100644 --- a/server/src/data/schema.ts +++ b/server/src/data/schema.ts @@ -206,6 +206,9 @@ const catalogues = pgTable("catalogues", { url: text("url").notNull().unique(), thumbnailUrl: text("thumbnail_url"), createdAt: timestamp("created_at").notNull().defaultNow(), + orgId: integer("org_id") + .notNull() + .references(() => orgs.id), }); const cataloguesRelations = relations(catalogues, ({ many }) => ({ @@ -468,6 +471,9 @@ const settings = pgTable("settings", { isEncrypted: boolean("is_encrypted").default(false).notNull(), encryptedPreview: text("encrypted_preview"), createdAt: timestamp("created_at").notNull().defaultNow(), + orgId: integer("org_id") + .notNull() + .references(() => orgs.id), }); const users = pgTable("users", { @@ -475,9 +481,35 @@ const users = pgTable("users", { name: text("name").notNull(), email: text("email").notNull().unique(), password: text("password").notNull(), + isStaff: boolean("is_staff").notNull().default(false), createdAt: timestamp("created_at").notNull().defaultNow(), }); +const orgs = pgTable("orgs", { + id: serial("id").primaryKey(), + name: text("name").notNull(), + logo: text("name").default("null"), + description: text("description").default("null"), + createdAt: timestamp("created_at").notNull().defaultNow(), +}); + +const roleTypes = pgEnum('role_type', [ + 'viewer', // Unused, reserved for future read-only use + 'member', // Default, can see organization data and run extractions + 'admin', // Same as member but can invite new or remove existing members +]); + +const memberships = pgTable("memberships", { + orgId: integer("org_id") + .notNull() + .references(() => orgs.id), + userId: integer("user_id") + .notNull() + .references(() => users.id), + role: roleTypes().default('member'), +}); + + export function encryptForDb(text: string) { const IV = randomBytes(16); let cipher = createCipheriv("aes-256-cbc", Buffer.from(ENCRYPTION_KEY), IV); @@ -520,4 +552,7 @@ export { recipesRelations, settings, users, + orgs, + memberships, + roleTypes, }; diff --git a/server/src/data/settings.ts b/server/src/data/settings.ts index 3550d47..1920cc1 100644 --- a/server/src/data/settings.ts +++ b/server/src/data/settings.ts @@ -1,15 +1,20 @@ import db from "../data"; import { encryptForDb, settings } from "../data/schema"; -export async function findSettings() { - return db.query.settings.findMany(); +export async function findSettings(orgId: number) { + return db.query.settings.findMany({ + where(fields, { eq }) { + return eq(fields?.orgId, orgId) + }, + }); } export async function createOrUpdate( key: string, value: string, isEncrypted: boolean = false, - encryptedPreview: string | null + encryptedPreview: string | null, + orgId: number, ) { value = isEncrypted ? encryptForDb(value) : value; return db @@ -19,6 +24,7 @@ export async function createOrUpdate( value, isEncrypted, encryptedPreview, + orgId, }) .onConflictDoUpdate({ target: settings.key, diff --git a/server/src/data/types.ts b/server/src/data/types.ts new file mode 100644 index 0000000..81301dd --- /dev/null +++ b/server/src/data/types.ts @@ -0,0 +1,4 @@ +import { InferSelectModel } from "drizzle-orm"; +import { orgs } from "./schema"; + +export type Organizations = InferSelectModel; \ No newline at end of file diff --git a/server/src/fastifySessionAuth.ts b/server/src/fastifySessionAuth.ts index 337ea77..5c58d07 100644 --- a/server/src/fastifySessionAuth.ts +++ b/server/src/fastifySessionAuth.ts @@ -27,6 +27,7 @@ const fastifySessionAuth: FastifyPluginCallback = fastifyPlugin( const user = await findUserById(parseInt(userId)); if (user) { req.user = user; + // add selected orgId to the user context } else { req.session.set("userId", undefined); } diff --git a/server/src/routers/catalogues.ts b/server/src/routers/catalogues.ts index 78c1cfb..c226257 100644 --- a/server/src/routers/catalogues.ts +++ b/server/src/routers/catalogues.ts @@ -11,6 +11,7 @@ import { } from "../data/catalogues"; import { findDatasets } from "../data/datasets"; import { fetchPreview } from "../extraction/browser"; +import { findOrgsByUser } from "@/data/orgs"; export const cataloguesRouter = router({ preview: publicProcedure @@ -35,7 +36,7 @@ export const cataloguesRouter = router({ return { totalItems, totalPages, - results: await findCatalogues(20, opts.input.page * 20 - 20), + results: await findCatalogues(opts.ctx.user.orgId, 20, opts.input.page * 20 - 20), }; }), create: publicProcedure @@ -48,7 +49,8 @@ export const cataloguesRouter = router({ ) .mutation(async (opts) => { const { name, url, thumbnailUrl } = opts.input; - const existingCatalogue = await findCatalogueByUrl(url); + const userOrgs = await findOrgsByUser(opts.ctx.user.id); + const existingCatalogue = await findCatalogueByUrl(url, userOrgs.map(org => org.orgId)); if (existingCatalogue) { return { id: existingCatalogue.id, diff --git a/server/src/routers/orgs.ts b/server/src/routers/orgs.ts new file mode 100644 index 0000000..52a28db --- /dev/null +++ b/server/src/routers/orgs.ts @@ -0,0 +1,10 @@ +import { z } from "zod"; +import { publicProcedure, router } from "."; +import { + findOrgsByUser +} from "../data/orgs"; + +export const orgsRouter = router({ + ofCurrentUser: publicProcedure + .query(({ ctx: { user }}) => findOrgsByUser(user!.id)), +}); diff --git a/server/src/routers/settings.ts b/server/src/routers/settings.ts index 27f6b57..5050483 100644 --- a/server/src/routers/settings.ts +++ b/server/src/routers/settings.ts @@ -1,10 +1,11 @@ import { z } from "zod"; import { publicProcedure, router } from "."; import { createOrUpdate, findSettings } from "../data/settings"; +import { findUserById } from '../data/users'; export const settingsRouter = router({ list: publicProcedure.query(async (_opts) => { - const allSettings = await findSettings(); + const allSettings = await findSettings(_opts.ctx.user?.orgId); return allSettings.map((setting) => ({ ...setting, value: setting.isEncrypted ? "*****" : setting.value, @@ -21,7 +22,8 @@ export const settingsRouter = router({ "OPENAI_API_KEY", opts.input.apiKey, true, - `sk-...${opts.input.apiKey.slice(-4)}` + `sk-...${opts.input.apiKey.slice(-4)}`, + opts.ctx.user.orgId ); }), }); diff --git a/server/src/trpcContext.ts b/server/src/trpcContext.ts index 8bc64ab..628a9d4 100644 --- a/server/src/trpcContext.ts +++ b/server/src/trpcContext.ts @@ -1,6 +1,7 @@ import { CreateFastifyContextOptions } from "@trpc/server/adapters/fastify"; export function createContext({ req, res }: CreateFastifyContextOptions) { - const user = (req as any).user; - return { req, res, user }; + const user = req.user; + const orgId = req.headers['x-ce-org-id']; + return { req, res, user, orgId }; } export type Context = Awaited>; diff --git a/server/tsconfig.json b/server/tsconfig.json index 949333f..510edf9 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -7,6 +7,7 @@ "lib": ["ES2022"], "outDir": "./dist", "strict": true, + "strictNullChecks": true, "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true,