diff --git a/DSL/Liquibase/langfuse-init/init-langfuse.sql b/DSL/Liquibase/langfuse-init/init-langfuse.sql new file mode 100644 index 0000000..25a815f --- /dev/null +++ b/DSL/Liquibase/langfuse-init/init-langfuse.sql @@ -0,0 +1,4 @@ +SELECT 'CREATE DATABASE "langfuse-db"' +WHERE NOT EXISTS ( + SELECT FROM pg_catalog.pg_database WHERE datname = 'langfuse-db' +)\gexec diff --git a/GUI/vite.config.ts.timestamp-1773932542024-ee7096644c66a.mjs b/GUI/vite.config.ts.timestamp-1773932542024-ee7096644c66a.mjs new file mode 100644 index 0000000..3ffe592 --- /dev/null +++ b/GUI/vite.config.ts.timestamp-1773932542024-ee7096644c66a.mjs @@ -0,0 +1,77 @@ +// vite.config.ts +import { defineConfig } from "file:///app/node_modules/vite/dist/node/index.js"; +import react from "file:///app/node_modules/@vitejs/plugin-react/dist/index.mjs"; +import tsconfigPaths from "file:///app/node_modules/vite-tsconfig-paths/dist/index.mjs"; +import svgr from "file:///app/node_modules/vite-plugin-svgr/dist/index.mjs"; +import path from "path"; + +// vitePlugin.js +function removeHiddenMenuItems(str) { + var _a, _b; + const badJson = str.replace("export default [", "[").replace("];", "]"); + const correctJson = badJson.replace(/(['"])?([a-z0-9A-Z_]+)(['"])?:/g, '"$2": '); + const isHiddenFeaturesEnabled = ((_a = process.env.REACT_APP_ENABLE_HIDDEN_FEATURES) == null ? void 0 : _a.toLowerCase().trim()) === "true" || ((_b = process.env.REACT_APP_ENABLE_HIDDEN_FEATURES) == null ? void 0 : _b.toLowerCase().trim()) === "1"; + const json = removeHidden(JSON.parse(correctJson), isHiddenFeaturesEnabled); + const updatedJson = JSON.stringify(json); + return "export default " + updatedJson + ";"; +} +function removeHidden(menuItems, isHiddenFeaturesEnabled) { + var _a; + if (!menuItems) + return menuItems; + const arr = (_a = menuItems == null ? void 0 : menuItems.filter((x) => !x.hidden)) == null ? void 0 : _a.filter((x) => isHiddenFeaturesEnabled || x.hiddenMode !== "production"); + for (const a of arr) { + a.children = removeHidden(a.children, isHiddenFeaturesEnabled); + } + return arr; +} + +// vite.config.ts +var __vite_injected_original_dirname = "/app"; +var vite_config_default = defineConfig({ + envPrefix: "REACT_APP_", + plugins: [ + react(), + tsconfigPaths(), + svgr(), + { + name: "removeHiddenMenuItemsPlugin", + transform: (str, id) => { + if (!id.endsWith("/menu-structure.json")) + return str; + return removeHiddenMenuItems(str); + } + } + ], + base: "/rag-search", + build: { + outDir: "./build", + target: "es2015", + emptyOutDir: true + }, + server: { + headers: { + ...process.env.REACT_APP_CSP && { + "Content-Security-Policy": process.env.REACT_APP_CSP + } + }, + allowedHosts: ["est-rag-rtc.rootcode.software", "localhost", "127.0.0.1"], + proxy: { + "/vault-agent-gui": { + target: "http://vault-agent-gui:8202", + changeOrigin: true, + rewrite: (path2) => path2.replace(/^\/vault-agent-gui/, "") + } + } + }, + resolve: { + alias: { + "~@fontsource": path.resolve(__vite_injected_original_dirname, "node_modules/@fontsource"), + "@": `${path.resolve(__vite_injected_original_dirname, "./src")}` + } + } +}); +export { + vite_config_default as default +}; +//# sourceMappingURL=data:application/json;base64,ewogICJ2ZXJzaW9uIjogMywKICAic291cmNlcyI6IFsidml0ZS5jb25maWcudHMiLCAidml0ZVBsdWdpbi5qcyJdLAogICJzb3VyY2VzQ29udGVudCI6IFsiY29uc3QgX192aXRlX2luamVjdGVkX29yaWdpbmFsX2Rpcm5hbWUgPSBcIi9hcHBcIjtjb25zdCBfX3ZpdGVfaW5qZWN0ZWRfb3JpZ2luYWxfZmlsZW5hbWUgPSBcIi9hcHAvdml0ZS5jb25maWcudHNcIjtjb25zdCBfX3ZpdGVfaW5qZWN0ZWRfb3JpZ2luYWxfaW1wb3J0X21ldGFfdXJsID0gXCJmaWxlOi8vL2FwcC92aXRlLmNvbmZpZy50c1wiO2ltcG9ydCB7IGRlZmluZUNvbmZpZyB9IGZyb20gJ3ZpdGUnO1xuaW1wb3J0IHJlYWN0IGZyb20gJ0B2aXRlanMvcGx1Z2luLXJlYWN0JztcbmltcG9ydCB0c2NvbmZpZ1BhdGhzIGZyb20gJ3ZpdGUtdHNjb25maWctcGF0aHMnO1xuaW1wb3J0IHN2Z3IgZnJvbSAndml0ZS1wbHVnaW4tc3Zncic7XG5pbXBvcnQgcGF0aCBmcm9tICdwYXRoJztcbmltcG9ydCB7IHJlbW92ZUhpZGRlbk1lbnVJdGVtcyB9IGZyb20gJy4vdml0ZVBsdWdpbic7XG5cbi8vIGh0dHBzOi8vdml0ZWpzLmRldi9jb25maWcvXG5leHBvcnQgZGVmYXVsdCBkZWZpbmVDb25maWcoe1xuICBlbnZQcmVmaXg6ICdSRUFDVF9BUFBfJyxcbiAgcGx1Z2luczogW1xuICAgIHJlYWN0KCksXG4gICAgdHNjb25maWdQYXRocygpLFxuICAgIHN2Z3IoKSxcbiAgICB7XG4gICAgICBuYW1lOiAncmVtb3ZlSGlkZGVuTWVudUl0ZW1zUGx1Z2luJyxcbiAgICAgIHRyYW5zZm9ybTogKHN0ciwgaWQpID0+IHtcbiAgICAgICAgaWYoIWlkLmVuZHNXaXRoKCcvbWVudS1zdHJ1Y3R1cmUuanNvbicpKVxuICAgICAgICAgIHJldHVybiBzdHI7XG4gICAgICAgIHJldHVybiByZW1vdmVIaWRkZW5NZW51SXRlbXMoc3RyKTtcbiAgICAgIH0sXG4gICAgfSxcbiAgXSxcbiAgYmFzZTogJy9yYWctc2VhcmNoJyxcbiAgYnVpbGQ6IHtcbiAgICBvdXREaXI6ICcuL2J1aWxkJyxcbiAgICB0YXJnZXQ6ICdlczIwMTUnLFxuICAgIGVtcHR5T3V0RGlyOiB0cnVlLFxuICB9LFxuICBzZXJ2ZXI6IHtcbiAgICBoZWFkZXJzOiB7XG4gICAgICAuLi4ocHJvY2Vzcy5lbnYuUkVBQ1RfQVBQX0NTUCAmJiB7XG4gICAgICAgICdDb250ZW50LVNlY3VyaXR5LVBvbGljeSc6IHByb2Nlc3MuZW52LlJFQUNUX0FQUF9DU1AsXG4gICAgICB9KSxcbiAgICB9LFxuICAgIGFsbG93ZWRIb3N0czogWydlc3QtcmFnLXJ0Yy5yb290Y29kZS5zb2Z0d2FyZScsICdsb2NhbGhvc3QnLCAnMTI3LjAuMC4xJ10sXG4gICAgcHJveHk6IHtcbiAgICAgICcvdmF1bHQtYWdlbnQtZ3VpJzoge1xuICAgICAgICB0YXJnZXQ6ICdodHRwOi8vdmF1bHQtYWdlbnQtZ3VpOjgyMDInLFxuICAgICAgICBjaGFuZ2VPcmlnaW46IHRydWUsXG4gICAgICAgIHJld3JpdGU6IChwYXRoKSA9PiBwYXRoLnJlcGxhY2UoL15cXC92YXVsdC1hZ2VudC1ndWkvLCAnJyksXG4gICAgICB9LFxuICAgIH0sXG4gIH0sXG4gIHJlc29sdmU6IHtcbiAgICBhbGlhczoge1xuICAgICAgJ35AZm9udHNvdXJjZSc6IHBhdGgucmVzb2x2ZShfX2Rpcm5hbWUsICdub2RlX21vZHVsZXMvQGZvbnRzb3VyY2UnKSxcbiAgICAgICdAJzogYCR7cGF0aC5yZXNvbHZlKF9fZGlybmFtZSwgJy4vc3JjJyl9YCxcbiAgICB9LFxuICB9LFxufSk7XG4iLCAiY29uc3QgX192aXRlX2luamVjdGVkX29yaWdpbmFsX2Rpcm5hbWUgPSBcIi9hcHBcIjtjb25zdCBfX3ZpdGVfaW5qZWN0ZWRfb3JpZ2luYWxfZmlsZW5hbWUgPSBcIi9hcHAvdml0ZVBsdWdpbi5qc1wiO2NvbnN0IF9fdml0ZV9pbmplY3RlZF9vcmlnaW5hbF9pbXBvcnRfbWV0YV91cmwgPSBcImZpbGU6Ly8vYXBwL3ZpdGVQbHVnaW4uanNcIjtleHBvcnQgZnVuY3Rpb24gcmVtb3ZlSGlkZGVuTWVudUl0ZW1zKHN0cikge1xuICBjb25zdCBiYWRKc29uID0gc3RyLnJlcGxhY2UoJ2V4cG9ydCBkZWZhdWx0IFsnLCAnWycpLnJlcGxhY2UoJ107JywgJ10nKTtcbiAgY29uc3QgY29ycmVjdEpzb24gPSBiYWRKc29uLnJlcGxhY2UoLyhbJ1wiXSk/KFthLXowLTlBLVpfXSspKFsnXCJdKT86L2csICdcIiQyXCI6ICcpO1xuXG4gY29uc3QgaXNIaWRkZW5GZWF0dXJlc0VuYWJsZWQgPSBcbiAgICBwcm9jZXNzLmVudi5SRUFDVF9BUFBfRU5BQkxFX0hJRERFTl9GRUFUVVJFUz8udG9Mb3dlckNhc2UoKS50cmltKCkgPT09ICd0cnVlJyB8fFxuICAgIHByb2Nlc3MuZW52LlJFQUNUX0FQUF9FTkFCTEVfSElEREVOX0ZFQVRVUkVTPy50b0xvd2VyQ2FzZSgpLnRyaW0oKSA9PT0gJzEnO1xuXG4gIGNvbnN0IGpzb24gPSByZW1vdmVIaWRkZW4oSlNPTi5wYXJzZShjb3JyZWN0SnNvbiksIGlzSGlkZGVuRmVhdHVyZXNFbmFibGVkKTtcbiAgXG4gIGNvbnN0IHVwZGF0ZWRKc29uID0gSlNPTi5zdHJpbmdpZnkoanNvbik7XG5cbiAgcmV0dXJuICdleHBvcnQgZGVmYXVsdCAnICsgdXBkYXRlZEpzb24gKyAnOydcbn1cblxuZnVuY3Rpb24gcmVtb3ZlSGlkZGVuKG1lbnVJdGVtcywgaXNIaWRkZW5GZWF0dXJlc0VuYWJsZWQpIHtcbiAgaWYoIW1lbnVJdGVtcykgcmV0dXJuIG1lbnVJdGVtcztcbiAgY29uc3QgYXJyID0gbWVudUl0ZW1zXG4gICAgPy5maWx0ZXIoeCA9PiAheC5oaWRkZW4pXG4gICAgPy5maWx0ZXIoeCA9PiBpc0hpZGRlbkZlYXR1cmVzRW5hYmxlZCB8fCB4LmhpZGRlbk1vZGUgIT09IFwicHJvZHVjdGlvblwiKTtcbiAgZm9yIChjb25zdCBhIG9mIGFycikge1xuICAgIGEuY2hpbGRyZW4gPSByZW1vdmVIaWRkZW4oYS5jaGlsZHJlbiwgaXNIaWRkZW5GZWF0dXJlc0VuYWJsZWQpO1xuICB9XG4gIHJldHVybiBhcnI7XG59XG4iXSwKICAibWFwcGluZ3MiOiAiO0FBQThMLFNBQVMsb0JBQW9CO0FBQzNOLE9BQU8sV0FBVztBQUNsQixPQUFPLG1CQUFtQjtBQUMxQixPQUFPLFVBQVU7QUFDakIsT0FBTyxVQUFVOzs7QUNKa0wsU0FBUyxzQkFBc0IsS0FBSztBQUF2TztBQUNFLFFBQU0sVUFBVSxJQUFJLFFBQVEsb0JBQW9CLEdBQUcsRUFBRSxRQUFRLE1BQU0sR0FBRztBQUN0RSxRQUFNLGNBQWMsUUFBUSxRQUFRLG1DQUFtQyxRQUFRO0FBRWhGLFFBQU0sNEJBQ0gsYUFBUSxJQUFJLHFDQUFaLG1CQUE4QyxjQUFjLFlBQVcsWUFDdkUsYUFBUSxJQUFJLHFDQUFaLG1CQUE4QyxjQUFjLFlBQVc7QUFFekUsUUFBTSxPQUFPLGFBQWEsS0FBSyxNQUFNLFdBQVcsR0FBRyx1QkFBdUI7QUFFMUUsUUFBTSxjQUFjLEtBQUssVUFBVSxJQUFJO0FBRXZDLFNBQU8sb0JBQW9CLGNBQWM7QUFDM0M7QUFFQSxTQUFTLGFBQWEsV0FBVyx5QkFBeUI7QUFmMUQ7QUFnQkUsTUFBRyxDQUFDO0FBQVcsV0FBTztBQUN0QixRQUFNLE9BQU0sNENBQ1IsT0FBTyxPQUFLLENBQUMsRUFBRSxZQURQLG1CQUVSLE9BQU8sT0FBSywyQkFBMkIsRUFBRSxlQUFlO0FBQzVELGFBQVcsS0FBSyxLQUFLO0FBQ25CLE1BQUUsV0FBVyxhQUFhLEVBQUUsVUFBVSx1QkFBdUI7QUFBQSxFQUMvRDtBQUNBLFNBQU87QUFDVDs7O0FEeEJBLElBQU0sbUNBQW1DO0FBUXpDLElBQU8sc0JBQVEsYUFBYTtBQUFBLEVBQzFCLFdBQVc7QUFBQSxFQUNYLFNBQVM7QUFBQSxJQUNQLE1BQU07QUFBQSxJQUNOLGNBQWM7QUFBQSxJQUNkLEtBQUs7QUFBQSxJQUNMO0FBQUEsTUFDRSxNQUFNO0FBQUEsTUFDTixXQUFXLENBQUMsS0FBSyxPQUFPO0FBQ3RCLFlBQUcsQ0FBQyxHQUFHLFNBQVMsc0JBQXNCO0FBQ3BDLGlCQUFPO0FBQ1QsZUFBTyxzQkFBc0IsR0FBRztBQUFBLE1BQ2xDO0FBQUEsSUFDRjtBQUFBLEVBQ0Y7QUFBQSxFQUNBLE1BQU07QUFBQSxFQUNOLE9BQU87QUFBQSxJQUNMLFFBQVE7QUFBQSxJQUNSLFFBQVE7QUFBQSxJQUNSLGFBQWE7QUFBQSxFQUNmO0FBQUEsRUFDQSxRQUFRO0FBQUEsSUFDTixTQUFTO0FBQUEsTUFDUCxHQUFJLFFBQVEsSUFBSSxpQkFBaUI7QUFBQSxRQUMvQiwyQkFBMkIsUUFBUSxJQUFJO0FBQUEsTUFDekM7QUFBQSxJQUNGO0FBQUEsSUFDQSxjQUFjLENBQUMsaUNBQWlDLGFBQWEsV0FBVztBQUFBLElBQ3hFLE9BQU87QUFBQSxNQUNMLG9CQUFvQjtBQUFBLFFBQ2xCLFFBQVE7QUFBQSxRQUNSLGNBQWM7QUFBQSxRQUNkLFNBQVMsQ0FBQ0EsVUFBU0EsTUFBSyxRQUFRLHNCQUFzQixFQUFFO0FBQUEsTUFDMUQ7QUFBQSxJQUNGO0FBQUEsRUFDRjtBQUFBLEVBQ0EsU0FBUztBQUFBLElBQ1AsT0FBTztBQUFBLE1BQ0wsZ0JBQWdCLEtBQUssUUFBUSxrQ0FBVywwQkFBMEI7QUFBQSxNQUNsRSxLQUFLLEdBQUcsS0FBSyxRQUFRLGtDQUFXLE9BQU8sQ0FBQztBQUFBLElBQzFDO0FBQUEsRUFDRjtBQUNGLENBQUM7IiwKICAibmFtZXMiOiBbInBhdGgiXQp9Cg== diff --git a/docker-compose-ec2.yml b/docker-compose-ec2.yml index cc48c1c..a2af178 100644 --- a/docker-compose-ec2.yml +++ b/docker-compose-ec2.yml @@ -103,11 +103,11 @@ services: container_name: resql image: resql depends_on: - rag_search_db: + rag-search-db: condition: service_started environment: - sqlms.datasources.[0].name=byk - - sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://rag_search_db:5432/rag-search #For LocalDb Use + - sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://rag-search-db:5432/rag-search #For LocalDb Use # sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://171.22.247.13:5435/byk?sslmode=require - sqlms.datasources.[0].username=postgres - sqlms.datasources.[0].password=dbadmin @@ -301,7 +301,7 @@ services: image: docker.io/langfuse/langfuse-worker:3 restart: always depends_on: &langfuse-depends-on - rag_search_db: + rag-search-db: condition: service_healthy minio: condition: service_healthy @@ -368,7 +368,7 @@ services: restart: always depends_on: - langfuse-worker - - rag_search_db + - rag-search-db ports: - 3005:3000 env_file: @@ -462,8 +462,8 @@ services: networks: - bykstack - rag_search_db: - container_name: rag_search_db + rag-search-db: + container_name: rag-search-db image: postgres:14.1 restart: always healthcheck: @@ -481,6 +481,7 @@ services: - 5436:5432 volumes: - rag-search-db:/var/lib/postgresql/data + - ./DSL/Liquibase/langfuse-init/init-langfuse.sql:/docker-entrypoint-initdb.d/init-langfuse.sql:ro networks: - bykstack diff --git a/docker-compose-test.yml b/docker-compose-test.yml index a9cfd5a..a0c5607 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -2,9 +2,9 @@ services: # === Core Infrastructure === # Shared PostgreSQL database (used by both application and Langfuse) - rag_search_db: + rag-search-db: image: postgres:14.1 - container_name: rag_search_db + container_name: rag-search-db restart: always environment: POSTGRES_USER: postgres @@ -89,11 +89,11 @@ services: container_name: resql image: ghcr.io/buerokratt/resql:v1.3.6 depends_on: - rag_search_db: + rag-search-db: condition: service_started environment: - sqlms.datasources.[0].name=byk - - sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://rag_search_db:5432/rag-search #For LocalDb Use + - sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://rag-search-db:5432/rag-search #For LocalDb Use # sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://171.22.247.13:5435/byk?sslmode=require - sqlms.datasources.[0].username=postgres - sqlms.datasources.[0].password=dbadmin @@ -222,7 +222,7 @@ services: container_name: langfuse-worker restart: always depends_on: - - rag_search_db + - rag-search-db - minio - redis - clickhouse @@ -230,7 +230,7 @@ services: - "127.0.0.1:3030:3030" environment: # Database - DATABASE_URL: postgresql://postgres:dbadmin@rag_search_db:5432/rag-search + DATABASE_URL: postgresql://postgres:dbadmin@rag-search-db:5432/rag-search # Auth & Security (TEST VALUES ONLY - NOT FOR PRODUCTION) # gitleaks:allow - These are test-only hex strings @@ -279,13 +279,13 @@ services: restart: always depends_on: - langfuse-worker - - rag_search_db + - rag-search-db - clickhouse ports: - "3000:3000" environment: # Database - DATABASE_URL: postgresql://postgres:dbadmin@rag_search_db:5432/rag-search + DATABASE_URL: postgresql://postgres:dbadmin@rag-search-db:5432/rag-search # Auth & Security (TEST VALUES ONLY - NOT FOR PRODUCTION) # gitleaks:allow - These are test-only hex strings diff --git a/docker-compose.yml b/docker-compose.yml index 0a82508..bf7f102 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -103,11 +103,11 @@ services: container_name: resql image: resql depends_on: - rag_search_db: + rag-search-db: condition: service_started environment: - sqlms.datasources.[0].name=byk - - sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://rag_search_db:5432/rag-search #For LocalDb Use + - sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://rag-search-db:5432/rag-search #For LocalDb Use # sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://171.22.247.13:5435/byk?sslmode=require - sqlms.datasources.[0].username=postgres - sqlms.datasources.[0].password=dbadmin @@ -249,7 +249,7 @@ services: image: docker.io/langfuse/langfuse-worker:3 restart: always depends_on: &langfuse-depends-on - rag_search_db: + rag-search-db: condition: service_healthy minio: condition: service_healthy @@ -316,7 +316,7 @@ services: restart: always depends_on: - langfuse-worker - - rag_search_db + - rag-search-db ports: - 3005:3000 env_file: @@ -410,8 +410,8 @@ services: networks: - bykstack - rag_search_db: - container_name: rag_search_db + rag-search-db: + container_name: rag-search-db image: postgres:14.1 restart: always healthcheck: @@ -429,6 +429,7 @@ services: - 5436:5432 volumes: - rag-search-db:/var/lib/postgresql/data + - ./DSL/Liquibase/langfuse-init/init-langfuse.sql:/docker-entrypoint-initdb.d/init-langfuse.sql:ro networks: - bykstack diff --git a/env.example b/env.example index 105b9f8..9bbc857 100644 --- a/env.example +++ b/env.example @@ -16,9 +16,9 @@ GF_USERS_ALLOW_SIGN_UP=false PORT=3000 POSTGRES_USER=postgres POSTGRES_PASSWORD=dbadmin -POSTGRES_DB=rag-search-langfuse +POSTGRES_DB=rag-search NEXTAUTH_URL=http://localhost:3005 -DATABASE_URL=postgresql://postgres:dbadmin@rag_search_db:5432/rag-search +DATABASE_URL=postgresql://postgres:dbadmin@rag-search-db:5432/langfuse-db SALT=changeme ENCRYPTION_KEY=changeme NEXTAUTH_SECRET=changeme diff --git a/kubernetes/CONTAINER_REGISTRY_SETUP.md b/kubernetes/CONTAINER_REGISTRY_SETUP.md new file mode 100644 index 0000000..d88f16b --- /dev/null +++ b/kubernetes/CONTAINER_REGISTRY_SETUP.md @@ -0,0 +1,35 @@ +# Container Registry Setup Guide + +This guide explains what components need to push to gcr + +## Overview + +The RAG Module consists of multiple container images that need to be pushed to your container registry. Currently, we use ECR for testing, but you should push images to your own registry before deployment. + + + +## Step 1: Build Container Images + +Build all required images from the repository root: + +### **1.1 GUI (Frontend)** + +```bash +cd GUI +docker build -t rag-module/gui:latest -f Dockerfile.dev . +cd .. +``` + +update the GUI helms values image: repository section with actual image + +### **1.2 LLM Orchestration Service** + +```bash +docker build -t rag-module/llm-orchestration-service:latest -f Dockerfile.llm_orchestration_service . +``` +update the LLM Orchestration Service helms values image: repository section with actual image (there are two places to update in this file) + +### **1.3 Authentication Layer** + + + diff --git a/kubernetes/Chart.lock b/kubernetes/Chart.lock new file mode 100644 index 0000000..47418e5 --- /dev/null +++ b/kubernetes/Chart.lock @@ -0,0 +1,84 @@ +dependencies: +- name: database + repository: file://./charts/database + version: 0.1.0 +- name: TIM-database + repository: file://./charts/TIM-database + version: 0.1.0 +- name: resql + repository: file://./charts/Resql + version: 0.1.0 +- name: ruuter-public + repository: file://./charts/Ruuter-Public + version: 0.1.0 +- name: ruuter-private + repository: file://./charts/Ruuter-Private + version: 0.1.0 +- name: data-mapper + repository: file://./charts/DataMapper + version: 0.1.0 +- name: TIM + repository: file://./charts/TIM + version: 0.1.0 +- name: Authentication-Layer + repository: file://./charts/Authentication-Layer + version: 0.1.0 +- name: CronManager + repository: file://./charts/CronManager + version: 0.1.0 +- name: GUI + repository: file://./charts/GUI + version: 0.1.0 +- name: Loki + repository: file://./charts/Loki + version: 0.1.0 +- name: Grafana + repository: file://./charts/Grafana + version: 0.1.0 +- name: S3-Ferry + repository: file://./charts/S3-Ferry + version: 0.1.0 +- name: minio + repository: file://./charts/minio + version: 0.1.0 +- name: Redis + repository: file://./charts/Redis + version: 0.1.0 +- name: Qdrant + repository: file://./charts/Qdrant + version: 0.1.0 +- name: ClickHouse + repository: file://./charts/ClickHouse + version: 0.1.0 +- name: Langfuse-Web + repository: file://./charts/Langfuse-Web + version: 0.1.0 +- name: Langfuse-Worker + repository: file://./charts/Langfuse-Worker + version: 0.1.0 +- name: Vault + repository: file://./charts/Vault + version: 0.1.0 +- name: Vault-Init + repository: file://./charts/Vault-Init + version: 0.1.0 +- name: Vault-Agent-GUI + repository: file://./charts/Vault-Agent-GUI + version: 0.1.0 +- name: Vault-Agent-Cron + repository: file://./charts/Vault-Agent-Cron + version: 0.1.0 +- name: Vault-Agent-LLM + repository: file://./charts/Vault-Agent-LLM + version: 0.1.0 +- name: LLM-Orchestration-Service + repository: file://./charts/LLM-Orchestration-Service + version: 0.1.0 +- name: Liquibase + repository: file://./charts/Liquibase + version: 0.1.0 +- name: Notifications-Node + repository: file://./charts/Notifications-Node + version: 0.1.0 +digest: sha256:48065436f01fcf7277161638c5fabe6c48afbcb1738e559ed03a921cd6a9d260 +generated: "2026-03-19T15:56:59.0549062+05:30" diff --git a/kubernetes/Chart.yaml b/kubernetes/Chart.yaml new file mode 100644 index 0000000..eb9a316 --- /dev/null +++ b/kubernetes/Chart.yaml @@ -0,0 +1,116 @@ +apiVersion: v2 +name: rag-module +description: Umbrella chart for RAG Module +version: 0.1.0 +type: application + +dependencies: + - name: database + version: 0.1.0 + repository: "file://./charts/database" + condition: database.enabled + - name: TIM-database + version: 0.1.0 + repository: "file://./charts/TIM-database" + condition: TIM-database.enabled + - name: resql + version: 0.1.0 + repository: "file://./charts/Resql" + condition: resql.enabled + - name: ruuter-public + version: 0.1.0 + repository: "file://./charts/Ruuter-Public" + condition: ruuter-public.enabled + - name: ruuter-private + version: 0.1.0 + repository: "file://./charts/Ruuter-Private" + condition: ruuter-private.enabled + - name: data-mapper + version: 0.1.0 + repository: "file://./charts/DataMapper" + condition: data-mapper.enabled + - name: TIM + version: 0.1.0 + repository: "file://./charts/TIM" + condition: TIM.enabled + - name: Authentication-Layer + version: 0.1.0 + repository: "file://./charts/Authentication-Layer" + condition: Authentication-Layer.enabled + - name: CronManager + version: 0.1.0 + repository: "file://./charts/CronManager" + condition: CronManager.enabled + - name: GUI + version: 0.1.0 + repository: "file://./charts/GUI" + condition: GUI.enabled + - name: Loki + version: 0.1.0 + repository: "file://./charts/Loki" + condition: Loki.enabled + - name: Grafana + version: 0.1.0 + repository: "file://./charts/Grafana" + condition: Grafana.enabled + - name: S3-Ferry + version: 0.1.0 + repository: "file://./charts/S3-Ferry" + condition: S3-Ferry.enabled + - name: minio + version: 0.1.0 + repository: "file://./charts/minio" + condition: minio.enabled + - name: Redis + version: 0.1.0 + repository: "file://./charts/Redis" + condition: Redis.enabled + - name: Qdrant + version: 0.1.0 + repository: "file://./charts/Qdrant" + condition: Qdrant.enabled + - name: ClickHouse + version: 0.1.0 + repository: "file://./charts/ClickHouse" + condition: ClickHouse.enabled + - name: Langfuse-Web + version: 0.1.0 + repository: "file://./charts/Langfuse-Web" + condition: Langfuse-Web.enabled + - name: Langfuse-Worker + version: 0.1.0 + repository: "file://./charts/Langfuse-Worker" + condition: Langfuse-Worker.enabled + - name: Vault + version: 0.1.0 + repository: "file://./charts/Vault" + condition: Vault.enabled + - name: Vault-Init + version: 0.1.0 + repository: "file://./charts/Vault-Init" + condition: Vault-Init.enabled + - name: Vault-Agent-GUI + version: 0.1.0 + repository: "file://./charts/Vault-Agent-GUI" + condition: Vault-Agent-GUI.enabled + - name: Vault-Agent-Cron + version: 0.1.0 + repository: "file://./charts/Vault-Agent-Cron" + condition: Vault-Agent-Cron.enabled + - name: Vault-Agent-LLM + version: 0.1.0 + repository: "file://./charts/Vault-Agent-LLM" + condition: Vault-Agent-LLM.enabled + - name: LLM-Orchestration-Service + version: 0.1.0 + repository: "file://./charts/LLM-Orchestration-Service" + condition: LLM-Orchestration-Service.enabled + - name: Liquibase + version: 0.1.0 + repository: "file://./charts/Liquibase" + condition: Liquibase.enabled + - name: Notifications-Node + version: 0.1.0 + repository: "file://./charts/Notifications-Node" + condition: Notifications-Node.enabled + diff --git a/kubernetes/LANGFUSE_SETUP.md b/kubernetes/LANGFUSE_SETUP.md new file mode 100644 index 0000000..6c0f11b --- /dev/null +++ b/kubernetes/LANGFUSE_SETUP.md @@ -0,0 +1,59 @@ +# Langfuse Setup + +**you can seed secrets in Langfuse-web , Langfuse-worker,clickhouse and database with .env file values** + +## 1. Verify Required Pods + +```bash +kubectl get pods -n your-namespace +``` + +All of the following must be `Running` or `Completed` — Langfuse will not start without them: + +| Pod | Purpose | +|---|---| +| `rag-search-db-0` | PostgreSQL (hosts `rag-search` and `langfuse-db`) | +| `minio-*` | Object storage for Langfuse events/media | +| `redis-*` | Queue backend for Langfuse worker | +| `clickhouse-*` | Analytics DB for Langfuse ingestion | +| `langfuse-worker-*` | Must be `Running` before web starts | +| `langfuse-web-*` | UI + runs DB migrations on first boot | +| `vault` | Secret storage | +| `vault-Init` | unseal vault | + +## 2. Wait for DB Migrations + +On first startup, `langfuse-web` runs database migrations — this takes 1–2 minutes. Watch the logs: + +```bash +kubectl logs -n your-namespace deployment/langfuse-web -f +``` + +Do **not** proceed until the pod is fully `Running`. + +## 3. Access the Dashboard + +```bash +kubectl port-forward -n your-namespace svc/langfuse-web 3005:3005 +``` + +Open **http://localhost:3005**, sign up / log in, then go to **Settings → API Keys → Create new key**. + +> Save both keys — the secret key is only shown once. +> - `pk-lf-...` → Public Key +> - `sk-lf-...` → Secret Key + +## 4. Store Keys in Vault + +```bash +kubectl cp store-langfuse-secrets.sh rag-module/vault-0:/tmp/store-langfuse-secrets.sh + +kubectl exec -n your-namespace vault-0 -- sh -c \ + "LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-lf-YOUR_KEY \ + LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-lf-YOUR_KEY \ + sh /tmp/store-langfuse-secrets.sh" +``` + +Replace `pk-lf-YOUR_KEY` and `sk-lf-YOUR_KEY` with the actual keys from step 3. + +The script stores them at `secret/data/langfuse/config` in Vault, where the LLM Orchestration Service reads them. diff --git a/kubernetes/charts/Langfuse-Web/templates/deployment-byk-langfuse-web.yaml b/kubernetes/charts/Langfuse-Web/templates/deployment-byk-langfuse-web.yaml index 403e253..18d1480 100644 --- a/kubernetes/charts/Langfuse-Web/templates/deployment-byk-langfuse-web.yaml +++ b/kubernetes/charts/Langfuse-Web/templates/deployment-byk-langfuse-web.yaml @@ -17,6 +17,31 @@ spec: app: "{{ .Values.release_name }}" component: langfuse-web spec: + initContainers: + - name: wait-for-clickhouse + image: busybox:1.35 + command: + - sh + - -c + - | + echo "Waiting for ClickHouse on port 9000..." + until nc -z clickhouse 9000 2>/dev/null; do + echo "ClickHouse not ready yet, retrying in 5s..." + sleep 5 + done + echo "ClickHouse is ready." + - name: wait-for-postgres + image: busybox:1.35 + command: + - sh + - -c + - | + echo "Waiting for PostgreSQL on port 5432..." + until nc -z rag-search-db 5432 2>/dev/null; do + echo "PostgreSQL not ready yet, retrying in 5s..." + sleep 5 + done + echo "PostgreSQL is ready." containers: - name: "{{ .Values.release_name }}" image: "{{ .Values.images.langfuse_web.registry }}/{{ .Values.images.langfuse_web.repository }}:{{ .Values.images.langfuse_web.tag }}" @@ -25,22 +50,16 @@ spec: - name: http containerPort: {{ .Values.service.targetPort }} protocol: TCP - # Non-sensitive env's from values.yaml env: {{- range $key, $value := .Values.env }} - name: {{ $key }} value: {{ $value | quote }} {{- end }} - # Sensitive env's from Kubernetes Secret - {{- if .Values.envFrom }} - envFrom: - {{- toYaml .Values.envFrom | nindent 12 }} - {{- end }} {{- if .Values.healthcheck.enabled }} livenessProbe: httpGet: path: /api/public/health - port: {{ .Values.service.port }} + port: {{ .Values.service.targetPort }} initialDelaySeconds: {{ .Values.healthcheck.initialDelaySeconds }} periodSeconds: {{ .Values.healthcheck.periodSeconds }} timeoutSeconds: {{ .Values.healthcheck.timeoutSeconds }} @@ -48,7 +67,7 @@ spec: readinessProbe: httpGet: path: /api/public/health - port: {{ .Values.service.port }} + port: {{ .Values.service.targetPort }} initialDelaySeconds: {{ .Values.healthcheck.initialDelaySeconds }} periodSeconds: {{ .Values.healthcheck.periodSeconds }} timeoutSeconds: {{ .Values.healthcheck.timeoutSeconds }} diff --git a/kubernetes/charts/database/Chart.lock b/kubernetes/charts/database/Chart.lock new file mode 100644 index 0000000..641f6d0 --- /dev/null +++ b/kubernetes/charts/database/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 12.2.6 +digest: sha256:6f50554d914d878d490c46307f120b87d39854e42f81411b13ffdd23aad21cb6 +generated: "2025-12-02T13:43:50.4497212+05:30" diff --git a/kubernetes/charts/database/Chart.yaml b/kubernetes/charts/database/Chart.yaml index 9612978..df3256a 100644 --- a/kubernetes/charts/database/Chart.yaml +++ b/kubernetes/charts/database/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: database description: PostgreSQL databases for RAG Module using pure PostgreSQL type: application -version: 0.2.0 +version: 0.1.0 \ No newline at end of file diff --git a/kubernetes/charts/database/templates/configmap.yaml b/kubernetes/charts/database/templates/configmap.yaml new file mode 100644 index 0000000..777a857 --- /dev/null +++ b/kubernetes/charts/database/templates/configmap.yaml @@ -0,0 +1,16 @@ +{{- range .Values.databases }} +{{- if .initdbScripts }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .name }}-initdb + labels: + app: {{ .name }} +data: + {{- range $filename, $content := .initdbScripts }} + {{ $filename }}: | + {{- $content | nindent 4 }} + {{- end }} +--- +{{- end }} +{{- end }} diff --git a/kubernetes/charts/database/templates/statefulset.yaml b/kubernetes/charts/database/templates/statefulset.yaml index 4ff6581..02194da 100644 --- a/kubernetes/charts/database/templates/statefulset.yaml +++ b/kubernetes/charts/database/templates/statefulset.yaml @@ -51,6 +51,16 @@ spec: volumeMounts: - name: data mountPath: /var/lib/postgresql/data + {{- if .initdbScripts }} + - name: initdb + mountPath: /docker-entrypoint-initdb.d + {{- end }} + {{- if .initdbScripts }} + volumes: + - name: initdb + configMap: + name: {{ .name }}-initdb + {{- end }} volumeClaimTemplates: - metadata: name: data diff --git a/kubernetes/charts/database/values.yaml b/kubernetes/charts/database/values.yaml index 8c43f0f..f8ecba9 100644 --- a/kubernetes/charts/database/values.yaml +++ b/kubernetes/charts/database/values.yaml @@ -5,6 +5,12 @@ databases: password: "{{ ragSearchDB.password }}" db: rag-search storage: 8Gi + initdbScripts: + init-langfuse.sql: | + SELECT 'CREATE DATABASE "langfuse-db"' + WHERE NOT EXISTS ( + SELECT FROM pg_catalog.pg_database WHERE datname = 'langfuse-db' + )\gexec - name: tim-postgresql username: tim password: "{{ TIMDB.password }}" @@ -23,3 +29,4 @@ persistence: storageClass: "" # specify your own accessModes: ["ReadWriteOnce"] + diff --git a/kubernetes/dashboard-admin.yaml b/kubernetes/dashboard-admin.yaml new file mode 100644 index 0000000..0485553 --- /dev/null +++ b/kubernetes/dashboard-admin.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admin-user + namespace: kubernetes-dashboard +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: admin-user +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: admin-user + namespace: kubernetes-dashboard \ No newline at end of file diff --git a/kubernetes/values.yaml b/kubernetes/values.yaml new file mode 100644 index 0000000..84f2ccf --- /dev/null +++ b/kubernetes/values.yaml @@ -0,0 +1,87 @@ +# Global configuration for RAG Module +global: + domain: "rag-module.local" + namespace: "rag-module" + storageClass: "local-path" + +# Individual service configurations +database: + enabled: true + +TIM-database: + enabled: false # tim-postgresql is now managed by the database chart + +resql: + enabled: true + +ruuter-public: + enabled: true + +ruuter-private: + enabled: true + +data-mapper: + enabled: true + +TIM: + enabled: true + +Authentication-Layer: + enabled: true + +CronManager: + enabled: true + +GUI: + enabled: true + +Loki: + enabled: true + +Grafana: + enabled: true + +S3-Ferry: + enabled: true + +minio: + enabled: true + +Redis: + enabled: true + +Qdrant: + enabled: true + +ClickHouse: + enabled: false + +Langfuse-Web: + enabled: false + +Langfuse-Worker: + enabled: false + +Vault: + enabled: true + +Vault-Init: + enabled: true + +Vault-Agent-LLM: + enabled: true + +Vault-Agent-GUI: + enabled: true + +Vault-Agent-Cron: + enabled: true + +LLM-Orchestration-Service: + enabled: true + +Liquibase: + enabled: true + +Notifications-Node: + enabled: true diff --git a/migrate.sh b/migrate.sh index c156698..8089cf1 100644 --- a/migrate.sh +++ b/migrate.sh @@ -12,4 +12,4 @@ INI_FILE="constants.ini" DB_PASSWORD=$(get_ini_value "$INI_FILE" "DB_PASSWORD") -docker run --rm --network bykstack -v `pwd`/DSL/Liquibase/changelog:/liquibase/changelog -v `pwd`/DSL/Liquibase/master.yml:/liquibase/master.yml -v `pwd`/DSL/Liquibase/data:/liquibase/data liquibase/liquibase:4.33 --defaultsFile=/liquibase/changelog/liquibase.properties --changelog-file=master.yml --url=jdbc:postgresql://rag_search_db:5432/rag-search?user=postgres --password=$DB_PASSWORD update +docker run --rm --network bykstack -v `pwd`/DSL/Liquibase/changelog:/liquibase/changelog -v `pwd`/DSL/Liquibase/master.yml:/liquibase/master.yml -v `pwd`/DSL/Liquibase/data:/liquibase/data liquibase/liquibase:4.33 --defaultsFile=/liquibase/changelog/liquibase.properties --changelog-file=master.yml --url=jdbc:postgresql://rag-search-db:5432/rag-search?user=postgres --password=$DB_PASSWORD update diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 333771a..9a348b2 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -491,7 +491,7 @@ def _run_database_migration(self) -> None: "liquibase/liquibase:4.33", "--defaultsFile=/liquibase/changelog/liquibase.properties", "--changelog-file=master.yml", - "--url=jdbc:postgresql://rag_search_db:5432/rag-search?user=postgres", + "--url=jdbc:postgresql://rag-search-db:5432/rag-search?user=postgres", "--password=dbadmin", "update", ], @@ -541,7 +541,7 @@ def _run_database_migration(self) -> None: "liquibase/liquibase:4.33", "--defaultsFile=/liquibase/changelog/liquibase.properties", "--changelog-file=master.yml", - "--url=jdbc:postgresql://rag_search_db:5432/rag-search?user=postgres", + "--url=jdbc:postgresql://rag-search-db:5432/rag-search?user=postgres", "--password=dbadmin", "update", ],