diff --git a/.dockerignore b/.dockerignore index 385a6449f..f66110780 100644 --- a/.dockerignore +++ b/.dockerignore @@ -28,6 +28,9 @@ yarn-error.log* # Node frontend/node_modules/ +frontend/.next/ +frontend/package-lock.json +frontend/tsconfig.tsbuildinfo node_modules/ .pnpm-store/ .pnpm-lock.yaml @@ -38,6 +41,7 @@ build/ # Backend backend/flower_db.sqlite +model-assets.tmp.*/ uploads/ test/ assets/ diff --git a/docker/.env.example b/.env.example similarity index 95% rename from docker/.env.example rename to .env.example index 3970efb95..bc5a96b8f 100644 --- a/docker/.env.example +++ b/.env.example @@ -81,8 +81,8 @@ MINIO_REGION=cn-north-1 MINIO_DEFAULT_BUCKET=nexent # Redis Config -REDIS_URL=redis://redis:6379/0 -REDIS_BACKEND_URL=redis://redis:6379/1 +REDIS_URL=redis://nexent-redis:6379/0 +REDIS_BACKEND_URL=redis://nexent-redis:6379/1 # Model Engine Config MODEL_ENGINE_ENABLED=false @@ -93,14 +93,14 @@ DASHBOARD_PASSWORD=Huawei123 # Supabase db Config SUPABASE_POSTGRES_PASSWORD=Huawei123 -SUPABASE_POSTGRES_HOST=db +SUPABASE_POSTGRES_HOST=nexent-supabase-db SUPABASE_POSTGRES_DB=supabase SUPABASE_POSTGRES_PORT=5436 # Supabase Auth Config SITE_URL=http://localhost:3011 -SUPABASE_URL=http://supabase-kong-mini:8000 -API_EXTERNAL_URL=http://supabase-kong-mini:8000 +SUPABASE_URL=http://nexent-supabase-kong:8000 +API_EXTERNAL_URL=http://nexent-supabase-kong:8000 DISABLE_SIGNUP=false JWT_EXPIRY=3600 DEBUG_JWT_EXPIRE_SECONDS=0 @@ -176,7 +176,7 @@ MONITORING_TRACE_MAX_CHARS=4000 MONITORING_TRACE_MAX_ITEMS=20 # Service name for identifying traces in observability platforms OTEL_SERVICE_NAME=nexent-backend -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 +OTEL_EXPORTER_OTLP_ENDPOINT=http://nexent-otel-collector:4318 # Optional signal-specific endpoints. Leave empty unless the backend requires them. OTEL_EXPORTER_OTLP_TRACES_ENDPOINT= OTEL_EXPORTER_OTLP_METRICS_ENDPOINT= @@ -222,7 +222,7 @@ WECHAT_OAUTH_APP_SECRET= # Base URL for OAuth callback (e.g., http://localhost:3000 for local dev) OAUTH_SSL_VERIFY=true OAUTH_CA_BUNDLE= -OAUTH_CALLBACK_BASE_URL=http://localhost:3000 +OAUTH_CALLBACK_BASE_URL=http://localhost:30000 # Asset owner role (opt-in; default false). Set true to enable ASSET_OWNER. ENABLE_ASSET_OWNER_ROLE=false @@ -231,7 +231,7 @@ ENABLE_ASSET_OWNER_ROLE=false CAS_ENABLED=false CAS_SERVER_URL= CAS_VALIDATE_PATH=/p3/serviceValidate -CAS_CALLBACK_BASE_URL=http://localhost:3000 +CAS_CALLBACK_BASE_URL=http://localhost:30000 # Supported values: # - disabled: disable CAS login entry and automatic CAS redirects. # - button: show CAS as an optional login entry. diff --git a/.github/workflows/auto-build-data-process-dev.yml b/.github/workflows/auto-build-data-process-dev.yml index 6be8bf638..42594242d 100644 --- a/.github/workflows/auto-build-data-process-dev.yml +++ b/.github/workflows/auto-build-data-process-dev.yml @@ -11,14 +11,18 @@ on: paths: - 'backend/**' - 'sdk/**' - - 'make/data_process/**' + - 'deploy/images/dockerfiles/data-process/**' + - 'deploy/common/**' + - 'deploy/sql/**' - '.github/workflows/**' push: branches: [develop, 'release/**', 'hotfix/**'] paths: - 'backend/**' - 'sdk/**' - - 'make/data_process/**' + - 'deploy/images/dockerfiles/data-process/**' + - 'deploy/common/**' + - 'deploy/sql/**' - '.github/workflows/**' jobs: @@ -35,7 +39,7 @@ jobs: rm -rf .git .gitattributes - name: Build data process image (amd64) and load locally run: | - docker build --platform linux/amd64 -t nexent/nexent-data-process:dev-amd64 -f make/data_process/Dockerfile . + docker build --platform linux/amd64 -t nexent/nexent-data-process:dev-amd64 -f deploy/images/dockerfiles/data-process/Dockerfile . build-data-process-arm64: runs-on: ubuntu-24.04-arm @@ -50,4 +54,4 @@ jobs: rm -rf .git .gitattributes - name: Build data process image (arm64) and load locally run: | - docker build --platform linux/arm64 -t nexent/nexent-data-process:dev-arm64 -f make/data_process/Dockerfile . \ No newline at end of file + docker build --platform linux/arm64 -t nexent/nexent-data-process:dev-arm64 -f deploy/images/dockerfiles/data-process/Dockerfile . \ No newline at end of file diff --git a/.github/workflows/auto-build-main-dev.yml b/.github/workflows/auto-build-main-dev.yml index 2815c50df..a667631b7 100644 --- a/.github/workflows/auto-build-main-dev.yml +++ b/.github/workflows/auto-build-main-dev.yml @@ -11,14 +11,18 @@ on: paths: - 'backend/**' - 'sdk/**' - - 'make/main/**' + - 'deploy/images/dockerfiles/main/**' + - 'deploy/common/**' + - 'deploy/sql/**' - '.github/workflows/**' push: branches: [develop, 'release/**', 'hotfix/**'] paths: - 'backend/**' - 'sdk/**' - - 'make/main/**' + - 'deploy/images/dockerfiles/main/**' + - 'deploy/common/**' + - 'deploy/sql/**' - '.github/workflows/**' jobs: @@ -29,7 +33,7 @@ jobs: uses: actions/checkout@v4 - name: Build main image (amd64) and load locally run: | - docker build --platform linux/amd64 -t nexent/nexent:dev-amd64 -f make/main/Dockerfile . + docker build --platform linux/amd64 -t nexent/nexent:dev-amd64 -f deploy/images/dockerfiles/main/Dockerfile . build-main-arm64: runs-on: ubuntu-24.04-arm @@ -38,4 +42,4 @@ jobs: uses: actions/checkout@v4 - name: Build main image (arm64) and load locally run: | - docker build --platform linux/arm64 -t nexent/nexent:dev-arm64 -f make/main/Dockerfile . \ No newline at end of file + docker build --platform linux/arm64 -t nexent/nexent:dev-arm64 -f deploy/images/dockerfiles/main/Dockerfile . \ No newline at end of file diff --git a/.github/workflows/auto-build-mcp-dev.yml b/.github/workflows/auto-build-mcp-dev.yml index 03aea08b2..a9a05e685 100644 --- a/.github/workflows/auto-build-mcp-dev.yml +++ b/.github/workflows/auto-build-mcp-dev.yml @@ -11,14 +11,14 @@ on: paths: - 'backend/**' - 'sdk/**' - - 'make/mcp/**' + - 'deploy/images/dockerfiles/mcp/**' - '.github/workflows/**' push: branches: [develop, 'release/**', 'hotfix/**'] paths: - 'backend/**' - 'sdk/**' - - 'make/mcp/**' + - 'deploy/images/dockerfiles/mcp/**' - '.github/workflows/**' jobs: @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@v4 - name: Build MCP image (amd64) and load locally run: | - docker build --platform linux/amd64 -t nexent/nexent-mcp:dev-amd64 -f make/mcp/Dockerfile . + docker build --platform linux/amd64 -t nexent/nexent-mcp:dev-amd64 -f deploy/images/dockerfiles/mcp/Dockerfile . build-mcp-arm64: runs-on: ubuntu-24.04-arm @@ -38,6 +38,6 @@ jobs: uses: actions/checkout@v4 - name: Build MCP image (arm64) and load locally run: | - docker build --platform linux/arm64 -t nexent/nexent-mcp:dev-arm64 -f make/mcp/Dockerfile . + docker build --platform linux/arm64 -t nexent/nexent-mcp:dev-arm64 -f deploy/images/dockerfiles/mcp/Dockerfile . diff --git a/.github/workflows/auto-build-terminal-dev.yml b/.github/workflows/auto-build-terminal-dev.yml index 62fc20165..81b5a9932 100644 --- a/.github/workflows/auto-build-terminal-dev.yml +++ b/.github/workflows/auto-build-terminal-dev.yml @@ -9,12 +9,12 @@ on: pull_request: branches: [develop, 'release/**', 'hotfix/**'] paths: - - 'make/terminal/**' + - 'deploy/images/dockerfiles/terminal/**' - '.github/workflows/**' push: branches: [develop, 'release/**', 'hotfix/**'] paths: - - 'make/terminal/**' + - 'deploy/images/dockerfiles/terminal/**' - '.github/workflows/**' jobs: @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v4 - name: Build terminal image (amd64) and load locally run: | - docker build --platform linux/amd64 -t nexent/nexent-ubuntu-terminal:dev-amd64 -f make/terminal/Dockerfile . + docker build --platform linux/amd64 -t nexent/nexent-ubuntu-terminal:dev-amd64 -f deploy/images/dockerfiles/terminal/Dockerfile . build-terminal-arm64: runs-on: ubuntu-24.04-arm @@ -34,4 +34,4 @@ jobs: uses: actions/checkout@v4 - name: Build terminal image (arm64) and load locally run: | - docker build --platform linux/arm64 -t nexent/nexent-ubuntu-terminal:dev-arm64 -f make/terminal/Dockerfile . \ No newline at end of file + docker build --platform linux/arm64 -t nexent/nexent-ubuntu-terminal:dev-arm64 -f deploy/images/dockerfiles/terminal/Dockerfile . \ No newline at end of file diff --git a/.github/workflows/auto-build-web-dev.yml b/.github/workflows/auto-build-web-dev.yml index a5abeb0b3..cd13fc4c8 100644 --- a/.github/workflows/auto-build-web-dev.yml +++ b/.github/workflows/auto-build-web-dev.yml @@ -10,13 +10,13 @@ on: branches: [develop, 'release/**', 'hotfix/**'] paths: - 'frontend/**' - - 'make/web/**' + - 'deploy/images/dockerfiles/web/**' - '.github/workflows/**' push: branches: [develop, 'release/**', 'hotfix/**'] paths: - 'frontend/**' - - 'make/web/**' + - 'deploy/images/dockerfiles/web/**' - '.github/workflows/**' jobs: @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v4 - name: Build web image (amd64) and load locally run: | - docker build --platform linux/amd64 -t nexent/nexent-web:dev-amd64 -f make/web/Dockerfile . + docker build --platform linux/amd64 -t nexent/nexent-web:dev-amd64 -f deploy/images/dockerfiles/web/Dockerfile . build-web-arm64: runs-on: ubuntu-24.04-arm @@ -36,4 +36,4 @@ jobs: uses: actions/checkout@v4 - name: Build web image (arm64) and load locally run: | - docker build --platform linux/arm64 -t nexent/nexent-web:dev-arm64 -f make/web/Dockerfile . \ No newline at end of file + docker build --platform linux/arm64 -t nexent/nexent-web:dev-arm64 -f deploy/images/dockerfiles/web/Dockerfile . \ No newline at end of file diff --git a/.github/workflows/auto-unit-test.yml b/.github/workflows/auto-unit-test.yml index dace8dab6..f572b14c1 100644 --- a/.github/workflows/auto-unit-test.yml +++ b/.github/workflows/auto-unit-test.yml @@ -11,11 +11,24 @@ on: description: 'runner array in json format (e.g. ["ubuntu-latest"] or ["self-hosted"])' required: false default: '["ubuntu-24.04-arm"]' + pytest_workers: + description: 'parallel test workers (auto=CPU count, 0=serial, N=fixed count)' + required: false + default: 'auto' + pytest_file_timeout: + description: 'per-file timeout in seconds (0=disabled)' + required: false + default: '300' pull_request: branches: [develop, 'release/**', 'hotfix/**'] paths: - 'backend/**' - 'sdk/**' + - 'deploy/common/**' + - 'deploy/tests/**' + - 'deploy/offline/**' + - 'deploy/docker/**' + - 'deploy/k8s/**' - 'test/**' - '.github/workflows/**' push: @@ -23,6 +36,11 @@ on: paths: - 'backend/**' - 'sdk/**' + - 'deploy/common/**' + - 'deploy/tests/**' + - 'deploy/offline/**' + - 'deploy/docker/**' + - 'deploy/k8s/**' - 'test/**' - '.github/workflows/**' @@ -33,6 +51,12 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Run deployment script tests + run: | + bash deploy/tests/test_common.sh + bash deploy/tests/test_sql_migrations.sh + bash deploy/tests/test_build_offline_package.sh + - name: Set up Python uses: actions/setup-python@v4 with: @@ -49,23 +73,25 @@ jobs: cd .. - name: Run all tests and collect coverage + env: + NEXENT_PYTEST_WORKERS: ${{ github.event.inputs.pytest_workers || 'auto' }} + NEXENT_PYTEST_FILE_TIMEOUT: ${{ github.event.inputs.pytest_file_timeout || '300' }} run: | source backend/.venv/bin/activate && python test/run_all_test.py TEST_EXIT_CODE=$? if [ -f "test/coverage.xml" ]; then - echo "✅ Coverage XML file generated successfully." + echo "Coverage XML file generated successfully." else - echo "❌ Coverage XML file not found." + echo "Coverage XML file not found." exit 1 fi - # Check if tests actually passed if [ $TEST_EXIT_CODE -ne 0 ]; then - echo "❌ Tests failed with exit code $TEST_EXIT_CODE" + echo "Tests failed with exit code $TEST_EXIT_CODE" exit $TEST_EXIT_CODE else - echo "✅ All tests passed successfully." + echo "All tests passed successfully." fi - name: Upload coverage to Codecov diff --git a/.github/workflows/build-offline-package.yml b/.github/workflows/build-offline-package.yml index 6619cf764..4dfe38faa 100644 --- a/.github/workflows/build-offline-package.yml +++ b/.github/workflows/build-offline-package.yml @@ -3,19 +3,49 @@ name: Build Offline Deployment Package on: workflow_dispatch: inputs: + version: + description: 'Image version tag, e.g. v2.2.0 or latest' + required: false + default: '' + platform: + description: 'Target platform' + required: false + default: 'amd64' + type: choice + options: + - amd64 + - arm64 + image_source: + description: 'Image source' + required: false + default: 'general' + type: choice + options: + - general + - mainland + components: + description: 'Deployment components CSV' + required: false + default: 'infrastructure,application' + target: + description: 'Package target' + required: false + default: 'all' + type: choice + options: + - docker + - k8s + - all include_source: description: 'Include source code in the package' required: false - default: true + default: false type: boolean jobs: build-offline-package: runs-on: ubuntu-latest - strategy: - matrix: - platform: [amd64, arm64] - + steps: - name: Free disk space uses: jlumbroso/free-disk-space@main @@ -30,18 +60,20 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - + - name: Set version and platform variables id: set-vars run: | - PLATFORM="${{ matrix.platform }}" + PLATFORM="${{ inputs.platform }}" REF_TYPE="${{ github.ref_type }}" REF_NAME="${{ github.ref_name }}" - - if [ "$REF_TYPE" = "tag" ]; then + + if [ -n "${{ inputs.version }}" ]; then + VERSION="${{ inputs.version }}" + elif [ "$REF_TYPE" = "tag" ]; then VERSION="$REF_NAME" elif [ "$REF_TYPE" = "branch" ]; then if [ "$REF_NAME" = "main" ]; then @@ -52,42 +84,42 @@ jobs: else VERSION="latest" fi - + echo "version=$VERSION" >> $GITHUB_OUTPUT echo "platform=$PLATFORM" >> $GITHUB_OUTPUT - echo "package-name=nexent-offline-${PLATFORM}-${VERSION}" >> $GITHUB_OUTPUT - + echo "package-name=nexent-offline-${{ inputs.target }}-${PLATFORM}-${VERSION}" >> $GITHUB_OUTPUT + - name: Build offline package run: | - chmod +x scripts/offline/build_offline_package.sh - - ./scripts/offline/build_offline_package.sh \ + chmod +x deploy/offline/build_offline_package.sh + + ./deploy/offline/build_offline_package.sh \ --version "${{ steps.set-vars.outputs.version }}" \ - --platform "${{ matrix.platform }}" \ + --platform "${{ steps.set-vars.outputs.platform }}" \ --output-dir ./offline-output \ - --include-source "${{ inputs.include_source }}" - - - - - name: Create ZIP package + --include-source "${{ inputs.include_source }}" \ + --image-source "${{ inputs.image_source }}" \ + --components "${{ inputs.components }}" \ + --target "${{ inputs.target }}" \ + --compress true + + + + - name: Show zip package run: | PACKAGE_NAME="${{ steps.set-vars.outputs.package-name }}" - - cd offline-output - zip -r "../${PACKAGE_NAME}.zip" . - cd .. - - echo "Package created: ${PACKAGE_NAME}.zip" - + + echo "Package created by build script: ${PACKAGE_NAME}.zip" + ls -lh "${PACKAGE_NAME}.zip" - + - name: Upload artifact uses: actions/upload-artifact@v4 with: name: ${{ steps.set-vars.outputs.package-name }} path: ${{ steps.set-vars.outputs.package-name }}.zip retention-days: 30 - + - name: Summary run: | echo "" @@ -95,11 +127,14 @@ jobs: echo "Offline Package Build Summary" echo "========================================" echo "Version: ${{ steps.set-vars.outputs.version }}" - echo "Platform: ${{ matrix.platform }}" + echo "Platform: ${{ steps.set-vars.outputs.platform }}" echo "Package: ${{ steps.set-vars.outputs.package-name }}.zip" + echo "Target: ${{ inputs.target }}" + echo "Components: ${{ inputs.components }}" + echo "Image source: ${{ inputs.image_source }}" echo "Ref Type: ${{ github.ref_type }}" echo "Ref Name: ${{ github.ref_name }}" echo "========================================" echo "" - echo "Package contents:" - unzip -l "${{ steps.set-vars.outputs.package-name }}.zip" | head -50 \ No newline at end of file + echo "Package directory:" + ls -l . diff --git a/.github/workflows/docker-build-push-mainland.yml b/.github/workflows/docker-build-push-mainland.yml index 8c215c7ec..b2ce9453e 100644 --- a/.github/workflows/docker-build-push-mainland.yml +++ b/.github/workflows/docker-build-push-mainland.yml @@ -4,14 +4,9 @@ on: workflow_dispatch: inputs: version: - description: 'Image version tag (e.g. v1.0.0 or latest)' + description: 'Image version tag (e.g. v2.2.0 or latest)' required: true default: 'latest' - push_latest: - description: 'Also push latest tag' - required: false - default: false - type: boolean runner_label_json: description: 'runner array in json format (e.g. ["ubuntu-latest"] or ["self-hosted"])' required: true @@ -23,395 +18,54 @@ on: - 'v*' jobs: - build-and-push-main-amd64: + build-and-push: runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} + strategy: + fail-fast: false + matrix: + image: [main, web, data-process, mcp, terminal] steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build main image (amd64) and load locally - run: | - docker buildx build --platform linux/amd64 --load -t ccr.ccs.tencentyun.com/nexent-hub/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 -f make/main/Dockerfile --build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua . - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Push main image (amd64) to Tencent Cloud - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 - - name: Tag main image (amd64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag ccr.ccs.tencentyun.com/nexent-hub/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 ccr.ccs.tencentyun.com/nexent-hub/nexent:amd64 - - name: Push latest main image (amd64) to Tencent Cloud - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent:amd64 + - name: Free disk space for data-process + if: matrix.image == 'data-process' + run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - build-and-push-main-arm64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - name: Checkout code uses: actions/checkout@v4 - - name: Build main image (arm64) and load locally - run: | - docker buildx build --platform linux/arm64 --load -t ccr.ccs.tencentyun.com/nexent-hub/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 -f make/main/Dockerfile --build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua . - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Push main image (arm64) to Tencent Cloud - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - - name: Tag main image (arm64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag ccr.ccs.tencentyun.com/nexent-hub/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 ccr.ccs.tencentyun.com/nexent-hub/nexent:arm64 - - name: Push latest main image (arm64) to Tencent Cloud - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent:arm64 - build-and-push-data-process-amd64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Free up disk space on GitHub runner - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Clone model - run: | - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/Nexent-AI/model-assets - cd ./model-assets - GIT_TRACE=1 GIT_CURL_VERBOSE=1 GIT_LFS_LOG=debug git lfs pull - rm -rf .git .gitattributes - - name: Build data process image (amd64) and load locally - run: | - docker buildx build --platform linux/amd64 --load -t ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 -f make/data_process/Dockerfile --build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua . - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Push data process image (amd64) to Tencent Cloud - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 - - name: Tag data process image (amd64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:amd64 - - name: Push latest data process image (amd64) to Tencent Cloud - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:amd64 + uses: docker/setup-buildx-action@v3 - build-and-push-data-process-arm64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Free up disk space on GitHub runner - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Clone model + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Clone model assets for data-process + if: matrix.image == 'data-process' run: | GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/Nexent-AI/model-assets - cd ./model-assets - GIT_TRACE=1 GIT_CURL_VERBOSE=1 GIT_LFS_LOG=debug git lfs pull + cd model-assets + git lfs pull rm -rf .git .gitattributes - - name: Build data process image (arm64) and load locally - run: | - docker buildx build --platform linux/arm64 --load -t ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 -f make/data_process/Dockerfile --build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua . - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Push data process image (arm64) to Tencent Cloud - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - - name: Tag data process image (arm64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:arm64 - - name: Push latest data process image (arm64) to Tencent Cloud - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:arm64 - - build-and-push-web-amd64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build web image (amd64) and load locally - run: | - docker buildx build --platform linux/amd64 --load -t ccr.ccs.tencentyun.com/nexent-hub/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 -f make/web/Dockerfile --build-arg MIRROR=https://registry.npmmirror.com --build-arg APK_MIRROR=tsinghua . - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Push web image (amd64) to Tencent Cloud - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 - - name: Tag web image (amd64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag ccr.ccs.tencentyun.com/nexent-hub/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 ccr.ccs.tencentyun.com/nexent-hub/nexent-web:amd64 - - name: Push latest web image (amd64) to Tencent Cloud - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-web:amd64 - - build-and-push-web-arm64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build web image (arm64) and load locally - run: | - docker buildx build --platform linux/arm64 --load -t ccr.ccs.tencentyun.com/nexent-hub/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 -f make/web/Dockerfile --build-arg MIRROR=https://registry.npmmirror.com --build-arg APK_MIRROR=tsinghua . - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Push web image (arm64) to Tencent Cloud - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - - name: Tag web image (arm64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag ccr.ccs.tencentyun.com/nexent-hub/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 ccr.ccs.tencentyun.com/nexent-hub/nexent-web:arm64 - - name: Push latest web image (arm64) to Tencent Cloud - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-web:arm64 - - build-and-push-terminal-amd64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build terminal image (amd64) and load locally - run: | - docker buildx build --platform linux/amd64 --load -t ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 -f make/terminal/Dockerfile . - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Push terminal image (amd64) to Tencent Cloud - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 - - name: Tag terminal image (amd64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:amd64 - - name: Push latest terminal image (amd64) to Tencent Cloud - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:amd64 - - build-and-push-terminal-arm64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build terminal image (arm64) and load locally - run: | - docker buildx build --platform linux/arm64 --load -t ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 -f make/terminal/Dockerfile . - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Push terminal image (arm64) to Tencent Cloud - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - - name: Tag terminal image (arm64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:arm64 - - name: Push latest terminal image (arm64) to Tencent Cloud - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:arm64 - - build-and-push-mcp-amd64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build MCP image (amd64) and load locally - run: | - docker buildx build --platform linux/amd64 --load -t ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 -f make/mcp/Dockerfile --build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua . - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Push MCP image (amd64) to Tencent Cloud - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 - - name: Tag MCP image (amd64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:amd64 - - name: Push latest MCP image (amd64) to Tencent Cloud - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:amd64 - build-and-push-mcp-arm64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx + - name: Resolve image version + id: version run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION="${{ github.event.inputs.version }}" + elif [ "${{ github.ref }}" = "refs/heads/main" ]; then + VERSION="latest" else - docker buildx use nexent_builder + VERSION="${{ github.ref_name }}" fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build MCP image (arm64) and load locally - run: | - docker buildx build --platform linux/arm64 --load -t ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 -f make/mcp/Dockerfile --build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua . - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Push MCP image (arm64) to Tencent Cloud - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - - name: Tag MCP image (arm64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:arm64 - - name: Push latest MCP image (arm64) to Tencent Cloud - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:arm64 + echo "value=$VERSION" >> "$GITHUB_OUTPUT" - manifest-push-main: - runs-on: ubuntu-latest - needs: - - build-and-push-main-amd64 - - build-and-push-main-arm64 - steps: - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Create and push manifest for main (Tencent Cloud) - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - run: | - docker manifest create ccr.ccs.tencentyun.com/nexent-hub/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} \ - ccr.ccs.tencentyun.com/nexent-hub/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 \ - ccr.ccs.tencentyun.com/nexent-hub/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - docker manifest push ccr.ccs.tencentyun.com/nexent-hub/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} - - name: Create and push latest manifest for main (Tencent Cloud) - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: | - docker manifest create ccr.ccs.tencentyun.com/nexent-hub/nexent:latest \ - ccr.ccs.tencentyun.com/nexent-hub/nexent:amd64 \ - ccr.ccs.tencentyun.com/nexent-hub/nexent:arm64 - docker manifest push ccr.ccs.tencentyun.com/nexent-hub/nexent:latest + run: echo "${{ secrets.TCR_PASSWORD }}" | docker login ccr.ccs.tencentyun.com --username="${{ secrets.TCR_USERNAME }}" --password-stdin - manifest-push-data-process: - runs-on: ubuntu-latest - needs: - - build-and-push-data-process-amd64 - - build-and-push-data-process-arm64 - steps: - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Create and push manifest for data-process (Tencent Cloud) - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - run: | - docker manifest create ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - docker manifest push ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} - - name: Create and push latest manifest for data-process (Tencent Cloud) - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: | - docker manifest create ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:latest \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:amd64 \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:arm64 - docker manifest push ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process:latest - - manifest-push-web: - runs-on: ubuntu-latest - needs: - - build-and-push-web-amd64 - - build-and-push-web-arm64 - steps: - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Create and push manifest for web (Tencent Cloud) - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - run: | - docker manifest create ccr.ccs.tencentyun.com/nexent-hub/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - docker manifest push ccr.ccs.tencentyun.com/nexent-hub/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} - - name: Create and push latest manifest for web (Tencent Cloud) - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: | - docker manifest create ccr.ccs.tencentyun.com/nexent-hub/nexent-web:latest \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-web:amd64 \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-web:arm64 - docker manifest push ccr.ccs.tencentyun.com/nexent-hub/nexent-web:latest - - manifest-push-terminal: - runs-on: ubuntu-latest - needs: - - build-and-push-terminal-amd64 - - build-and-push-terminal-arm64 - steps: - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Create and push manifest for terminal (Tencent Cloud) - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - run: | - docker manifest create ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - docker manifest push ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} - - name: Create and push latest manifest for terminal (Tencent Cloud) - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: | - docker manifest create ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:latest \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:amd64 \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:arm64 - docker manifest push ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:latest - - manifest-push-mcp: - runs-on: ubuntu-latest - needs: - - build-and-push-mcp-amd64 - - build-and-push-mcp-arm64 - steps: - - name: Login to Tencent Cloud - run: echo ${{ secrets.TCR_PASSWORD }} | docker login ccr.ccs.tencentyun.com --username=${{ secrets.TCR_USERNAME }} --password-stdin - - name: Create and push manifest for mcp (Tencent Cloud) - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - run: | - docker manifest create ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - docker manifest push ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} - - name: Create and push latest manifest for mcp (Tencent Cloud) - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') + - name: Build and push run: | - docker manifest create ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:latest \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:amd64 \ - ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:arm64 - docker manifest push ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp:latest \ No newline at end of file + bash deploy/images/build.sh \ + --image "${{ matrix.image }}" \ + --platform "linux/amd64,linux/arm64" \ + --version "${{ steps.version.outputs.value }}" \ + --registry mainland \ + --push diff --git a/.github/workflows/docker-build-push-overseas.yml b/.github/workflows/docker-build-push-overseas.yml index dcbe9d642..ea02dd410 100644 --- a/.github/workflows/docker-build-push-overseas.yml +++ b/.github/workflows/docker-build-push-overseas.yml @@ -4,14 +4,9 @@ on: workflow_dispatch: inputs: version: - description: 'Image version tag (e.g. v1.0.0 or latest)' + description: 'Image version tag (e.g. v2.2.0 or latest)' required: true default: 'latest' - push_latest: - description: 'Also push latest tag' - required: false - default: false - type: boolean runner_label_json: description: 'runner array in json format (e.g. ["ubuntu-latest"] or ["self-hosted"])' required: true @@ -23,395 +18,54 @@ on: - 'v*' jobs: - build-and-push-main-amd64: + build-and-push: runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} + strategy: + fail-fast: false + matrix: + image: [main, web, data-process, mcp, terminal] steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build main image (amd64) and load locally - run: | - docker buildx build --platform linux/amd64 -t nexent/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 --load -f make/main/Dockerfile . - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Push main image (amd64) to DockerHub - run: docker push nexent/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 - - name: Tag main image (amd64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag nexent/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 nexent/nexent:amd64 - - name: Push latest main image (amd64) to DockerHub - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push nexent/nexent:amd64 + - name: Free disk space for data-process + if: matrix.image == 'data-process' + run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - build-and-push-main-arm64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - name: Checkout code uses: actions/checkout@v4 - - name: Build main image (arm64) and load locally - run: | - docker buildx build --platform linux/arm64 -t nexent/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 --load -f make/main/Dockerfile . - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Push main image (arm64) to DockerHub - run: docker push nexent/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - - name: Tag main image (arm64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag nexent/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 nexent/nexent:arm64 - - name: Push latest main image (arm64) to DockerHub - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push nexent/nexent:arm64 - build-and-push-data-process-amd64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Free up disk space on GitHub runner - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Clone model - run: | - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/Nexent-AI/model-assets - cd ./model-assets - GIT_TRACE=1 GIT_CURL_VERBOSE=1 GIT_LFS_LOG=debug git lfs pull - rm -rf .git .gitattributes - - name: Build data process image (amd64) and load locally - run: | - docker buildx build --platform linux/amd64 -t nexent/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 --load -f make/data_process/Dockerfile . - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Push data process image (amd64) to DockerHub - run: docker push nexent/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 - - name: Tag data process image (amd64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag nexent/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 nexent/nexent-data-process:amd64 - - name: Push latest data process image (amd64) to DockerHub - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push nexent/nexent-data-process:amd64 + uses: docker/setup-buildx-action@v3 - build-and-push-data-process-arm64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Free up disk space on GitHub runner - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Clone model + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Clone model assets for data-process + if: matrix.image == 'data-process' run: | GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/Nexent-AI/model-assets - cd ./model-assets - GIT_TRACE=1 GIT_CURL_VERBOSE=1 GIT_LFS_LOG=debug git lfs pull + cd model-assets + git lfs pull rm -rf .git .gitattributes - - name: Build data process image (arm64) and load locally - run: | - docker buildx build --platform linux/arm64 -t nexent/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 --load -f make/data_process/Dockerfile . - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Push data process image (arm64) to DockerHub - run: docker push nexent/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - - name: Tag data process image (arm64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag nexent/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 nexent/nexent-data-process:arm64 - - name: Push latest data process image (arm64) to DockerHub - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push nexent/nexent-data-process:arm64 - - build-and-push-web-amd64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build web image (amd64) and load locally - run: | - docker buildx build --platform linux/amd64 -t nexent/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 --load -f make/web/Dockerfile . - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Push web image (amd64) to DockerHub - run: docker push nexent/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 - - name: Tag web image (amd64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag nexent/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 nexent/nexent-web:amd64 - - name: Push latest web image (amd64) to DockerHub - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push nexent/nexent-web:amd64 - - build-and-push-web-arm64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build web image (arm64) and load locally - run: | - docker buildx build --platform linux/arm64 -t nexent/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 --load -f make/web/Dockerfile . - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Push web image (arm64) to DockerHub - run: docker push nexent/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - - name: Tag web image (arm64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag nexent/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 nexent/nexent-web:arm64 - - name: Push latest web image (arm64) to DockerHub - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push nexent/nexent-web:arm64 - - build-and-push-terminal-amd64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build terminal image (amd64) and load locally - run: | - docker buildx build --platform linux/amd64 -t nexent/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 --load -f make/terminal/Dockerfile . - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Push terminal image (amd64) to DockerHub - run: docker push nexent/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 - - name: Tag terminal image (amd64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag nexent/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 nexent/nexent-ubuntu-terminal:amd64 - - name: Push latest terminal image (amd64) to DockerHub - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push nexent/nexent-ubuntu-terminal:amd64 - - build-and-push-terminal-arm64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build terminal image (arm64) and load locally - run: | - docker buildx build --platform linux/arm64 -t nexent/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 --load -f make/terminal/Dockerfile . - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Push terminal image (arm64) to DockerHub - run: docker push nexent/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - - name: Tag terminal image (arm64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag nexent/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 nexent/nexent-ubuntu-terminal:arm64 - - name: Push latest terminal image (arm64) to DockerHub - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push nexent/nexent-ubuntu-terminal:arm64 - - build-and-push-mcp-amd64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx - run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use - else - docker buildx use nexent_builder - fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build MCP image (amd64) and load locally - run: | - docker buildx build --platform linux/amd64 -t nexent/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 --load -f make/mcp/Dockerfile . - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Push MCP image (amd64) to DockerHub - run: docker push nexent/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 - - name: Tag MCP image (amd64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag nexent/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 nexent/nexent-mcp:amd64 - - name: Push latest MCP image (amd64) to DockerHub - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push nexent/nexent-mcp:amd64 - build-and-push-mcp-arm64: - runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }} - steps: - - name: Set up Docker Buildx + - name: Resolve image version + id: version run: | - if ! docker buildx inspect nexent_builder > /dev/null 2>&1; then - docker buildx create --name nexent_builder --use + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION="${{ github.event.inputs.version }}" + elif [ "${{ github.ref }}" = "refs/heads/main" ]; then + VERSION="latest" else - docker buildx use nexent_builder + VERSION="${{ github.ref_name }}" fi - - name: Checkout code - uses: actions/checkout@v4 - - name: Build MCP image (arm64) and load locally - run: | - docker buildx build --platform linux/arm64 -t nexent/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 --load -f make/mcp/Dockerfile . - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Push MCP image (arm64) to DockerHub - run: docker push nexent/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - - name: Tag MCP image (arm64) as latest - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker tag nexent/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 nexent/nexent-mcp:arm64 - - name: Push latest MCP image (arm64) to DockerHub - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: docker push nexent/nexent-mcp:arm64 + echo "value=$VERSION" >> "$GITHUB_OUTPUT" - manifest-push-main: - runs-on: ubuntu-latest - needs: - - build-and-push-main-amd64 - - build-and-push-main-arm64 - steps: - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Create and push manifest for main (DockerHub) - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - run: | - docker manifest create nexent/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} \ - nexent/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 \ - nexent/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - docker manifest push nexent/nexent:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} - - name: Create and push latest manifest for main (DockerHub) - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: | - docker manifest create nexent/nexent:latest \ - nexent/nexent:amd64 \ - nexent/nexent:arm64 - docker manifest push nexent/nexent:latest + run: echo "${{ secrets.DOCKERHUB_TOKEN }}" | docker login -u nexent --password-stdin - manifest-push-data-process: - runs-on: ubuntu-latest - needs: - - build-and-push-data-process-amd64 - - build-and-push-data-process-arm64 - steps: - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Create and push manifest for data-process (DockerHub) - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - run: | - docker manifest create nexent/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} \ - nexent/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 \ - nexent/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - docker manifest push nexent/nexent-data-process:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} - - name: Create and push latest manifest for data-process (DockerHub) - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: | - docker manifest create nexent/nexent-data-process:latest \ - nexent/nexent-data-process:amd64 \ - nexent/nexent-data-process:arm64 - docker manifest push nexent/nexent-data-process:latest - - manifest-push-web: - runs-on: ubuntu-latest - needs: - - build-and-push-web-amd64 - - build-and-push-web-arm64 - steps: - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Create and push manifest for web (DockerHub) - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - run: | - docker manifest create nexent/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} \ - nexent/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 \ - nexent/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - docker manifest push nexent/nexent-web:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} - - name: Create and push latest manifest for web (DockerHub) - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: | - docker manifest create nexent/nexent-web:latest \ - nexent/nexent-web:amd64 \ - nexent/nexent-web:arm64 - docker manifest push nexent/nexent-web:latest - - manifest-push-terminal: - runs-on: ubuntu-latest - needs: - - build-and-push-terminal-amd64 - - build-and-push-terminal-arm64 - steps: - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Create and push manifest for terminal (DockerHub) - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - run: | - docker manifest create nexent/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} \ - nexent/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 \ - nexent/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - docker manifest push nexent/nexent-ubuntu-terminal:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} - - name: Create and push latest manifest for terminal (DockerHub) - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') - run: | - docker manifest create nexent/nexent-ubuntu-terminal:latest \ - nexent/nexent-ubuntu-terminal:amd64 \ - nexent/nexent-ubuntu-terminal:arm64 - docker manifest push nexent/nexent-ubuntu-terminal:latest - - manifest-push-mcp: - runs-on: ubuntu-latest - needs: - - build-and-push-mcp-amd64 - - build-and-push-mcp-arm64 - steps: - - name: Login to DockerHub - run: echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login -u nexent --password-stdin - - name: Create and push manifest for mcp (DockerHub) - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - run: | - docker manifest create nexent/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} \ - nexent/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-amd64 \ - nexent/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }}-arm64 - docker manifest push nexent/nexent-mcp:${{ github.event.inputs.version || (github.event_name == 'push' && github.ref == 'refs/heads/main' && 'latest') || github.ref_name }} - - name: Create and push latest manifest for mcp (DockerHub) - if: (github.event.inputs.push_latest == 'true' && github.event_name == 'workflow_dispatch') || (github.event_name == 'push' && github.ref == 'refs/heads/main') + - name: Build and push run: | - docker manifest create nexent/nexent-mcp:latest \ - nexent/nexent-mcp:amd64 \ - nexent/nexent-mcp:arm64 - docker manifest push nexent/nexent-mcp:latest \ No newline at end of file + bash deploy/images/build.sh \ + --image "${{ matrix.image }}" \ + --platform "linux/amd64,linux/arm64" \ + --version "${{ steps.version.outputs.value }}" \ + --registry general \ + --push diff --git a/.github/workflows/docker-deploy.yml b/.github/workflows/docker-deploy.yml index a77c2491f..709a2e667 100644 --- a/.github/workflows/docker-deploy.yml +++ b/.github/workflows/docker-deploy.yml @@ -28,7 +28,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Build main application image - run: docker build --build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua -t nexent/nexent:${{ github.event.inputs.app_version }} -t nexent/nexent -f make/main/Dockerfile . + run: docker build --build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua -t nexent/nexent:${{ github.event.inputs.app_version }} -t nexent/nexent -f deploy/images/dockerfiles/main/Dockerfile . build-data-process: runs-on: ${{ fromJson(inputs.runner_label_json) }} @@ -55,7 +55,7 @@ jobs: GIT_TRACE=1 GIT_CURL_VERBOSE=1 GIT_LFS_LOG=debug git lfs pull rm -rf .git .gitattributes - name: Build data process image - run: docker build --build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua -t nexent/nexent-data-process:${{ github.event.inputs.app_version }} -t nexent/nexent-data-process -f make/data_process/Dockerfile . + run: docker build --build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua -t nexent/nexent-data-process:${{ github.event.inputs.app_version }} -t nexent/nexent-data-process -f deploy/images/dockerfiles/data-process/Dockerfile . build-web: runs-on: ${{ fromJson(inputs.runner_label_json) }} @@ -63,7 +63,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Build web frontend image - run: docker build --build-arg MIRROR=https://registry.npmmirror.com --build-arg APK_MIRROR=tsinghua -t nexent/nexent-web:${{ github.event.inputs.app_version }} -t nexent/nexent-web -f make/web/Dockerfile . + run: docker build --build-arg MIRROR=https://registry.npmmirror.com --build-arg APK_MIRROR=tsinghua -t nexent/nexent-web:${{ github.event.inputs.app_version }} -t nexent/nexent-web -f deploy/images/dockerfiles/web/Dockerfile . build-docs: runs-on: ${{ fromJson(inputs.runner_label_json) }} @@ -71,7 +71,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Build docs image - run: docker build --progress=plain -t nexent/nexent-docs:${{ github.event.inputs.app_version }} -t nexent/nexent-docs -f make/docs/Dockerfile . + run: docker build --progress=plain -t nexent/nexent-docs:${{ github.event.inputs.app_version }} -t nexent/nexent-docs -f deploy/images/dockerfiles/docs/Dockerfile . deploy: runs-on: ${{ fromJson(inputs.runner_label_json) }} @@ -86,26 +86,26 @@ jobs: cp -r $GITHUB_WORKSPACE/* $HOME/nexent/ - name: Force APP_VERSION to latest in deploy.sh (CI only) run: | - sed -i 's/APP_VERSION="$(get_app_version)"/APP_VERSION="${{ github.event.inputs.app_version }}"/' $HOME/nexent/docker/deploy.sh + sed -i 's/APP_VERSION="$(get_app_version)"/APP_VERSION="${{ github.event.inputs.app_version }}"/' $HOME/nexent/deploy/docker/deploy.sh - name: Start docs container run: | docker stop nexent-docs 2>/dev/null || true docker rm nexent-docs 2>/dev/null || true docker run -d --name nexent-docs -p 4173:4173 nexent/nexent-docs - name: Ensure deploy.sh is executable - run: chmod +x $HOME/nexent/docker/deploy.sh + run: chmod +x $HOME/nexent/deploy.sh $HOME/nexent/deploy/docker/deploy.sh - name: Deploy with deploy.sh env: DEPLOYMENT_MODE: ${{ github.event.inputs.deployment_mode }} run: | - cd $HOME/nexent/docker + cd $HOME/nexent cp .env.example .env sed -i "s/APPID=.*/APPID=${{ secrets.VOICE_APPID }}/" .env sed -i "s/TOKEN=.*/TOKEN=${{ secrets.VOICE_TOKEN }}/" .env if [ "$DEPLOYMENT_MODE" = "production" ]; then - ./deploy.sh --mode 3 --is-mainland N --enable-terminal N --version 2 --root-dir "$HOME/nexent-production-data" + ./deploy.sh docker --mode 3 --is-mainland N --enable-terminal N --version 2 --root-dir "$HOME/nexent-production-data" else - ./deploy.sh --mode 1 --is-mainland N --enable-terminal N --version 2 --root-dir "$HOME/nexent-development-data" + ./deploy.sh docker --mode 1 --is-mainland N --enable-terminal N --version 2 --root-dir "$HOME/nexent-development-data" fi diff --git a/.gitignore b/.gitignore index e0bac2b47..9a89d1dcd 100644 --- a/.gitignore +++ b/.gitignore @@ -19,16 +19,29 @@ docker/uploads docker/openssh-server docker/volumes/db/data docker/.env -docker/monitoring/monitoring.env +docker/.env.generated +deploy/docker/assets/monitoring/monitoring.env docker/.run docker/deploy.options k8s/helm/deploy.options scripts/deployment/local-config.yaml scripts/deployment/generated/ -docker/.env.generated -docker/docker-compose.generated.yml k8s/helm/nexent/generated-values.yaml +k8s/helm/nexent/generated-runtime-values.yaml k8s/helm/nexent/generated-secrets-values.yaml +k8s/helm/nexent/generated-persistence-values.yaml +deploy/docker/deploy.options +deploy/docker/openssh-server +deploy/k8s/deploy.options +deploy/common/local-config.yaml +deploy/common/generated/ +deploy/docker/.env.generated +deploy/docker/compose/docker-compose.generated.yml +deploy/k8s/helm/nexent/generated-values.yaml +deploy/k8s/helm/nexent/generated-runtime-values.yaml +deploy/k8s/helm/nexent/generated-secrets-values.yaml +deploy/k8s/helm/nexent/generated-persistence-values.yaml +offline-package/ frontend_standalone/ .pnpm-store/ @@ -53,8 +66,8 @@ logs/ .agents/ .devspace/ devspace.yaml -k8s/helm/**/*.tgz -k8s/helm/nexent/Chart.lock +deploy/k8s/helm/**/*.tgz +deploy/k8s/helm/nexent/Chart.lock MAC_DEVELOPMENT_GUIDE.md data/ @@ -65,5 +78,6 @@ sdk/benchmark/.env .pytest-tmp doc/mermaid +_doc/ -.claude/skills/python-import-triage \ No newline at end of file +.claude/skills/python-import-triage diff --git a/AGENTS.md b/AGENTS.md index 7798227b1..a631eb50f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ -When users ask you to perform tasks, check if any of the available skills below can help complete the task more effectively. Skills provide specialized capabilities and domain knowledge. +When users ask to perform tasks, check if any of the available skills below can help complete the task more effectively. Skills provide specialized capabilities and domain knowledge. How to use skills: - Invoke: `npx openskills read ` (run in your shell) @@ -40,3 +40,129 @@ Usage notes: + +--- + +## Project Overview + +Nexent is a zero-code platform for auto-generating AI agents. Monorepo with: +- `backend/` - FastAPI HTTP API +- `sdk/nexent/` - Core agent framework (pip package) +- `frontend/` - Next.js web UI +- `docker/` & `k8s/` - Deployment configs + +--- + +## Developer Commands + +### Backend (Python 3.10) + +```bash +# Setup +cd backend && uv sync --extra data-process --extra test + +# Install SDK for development +cd backend && uv pip install -e "../sdk[dev]" +``` + +### Run Tests + +```bash +# From project root, with backend venv activated +source backend/.venv/bin/activate && python test/run_all_test.py + +# Single test file +pytest test/backend/apps/test_agent_app.py -v +``` + +### Frontend (Next.js) + +```bash +cd frontend +npm run dev # Development server +npm run check-all # type-check + lint + format + build +``` + +### Docker Deployment + +```bash +cd docker +cp .env.example .env # Fill required configs +bash deploy.sh # Interactive deployment +``` + +--- + +## Architecture + +### Environment Variables + +**Single source of truth**: `backend/consts/const.py` + +- NO direct `os.getenv()` / `os.environ.get()` outside this file +- SDK (`sdk/nexent/`) NEVER reads env vars - accepts config via parameters +- Services read from `consts.const` and pass to SDK + +### Backend Layer Structure + +| Layer | Path | Responsibility | +|-------|------|----------------| +| Apps | `backend/apps/` | HTTP boundary: parse input, call services, map exceptions to HTTP | +| Services | `backend/services/` | Business logic orchestration, raise domain exceptions | +| Consts | `backend/consts/` | Env vars (`const.py`), exceptions (`exceptions.py`), error codes | + +**Exception flow**: Services raise domain exceptions → Apps map to HTTP status codes + +--- + +## Database Migrations + +**Location**: `docker/sql/*.sql` (versioned migration scripts) + +**Critical rule**: When adding columns/tables via migration script: +- Update `docker/init.sql` (Docker Compose fresh deploy) +- Update `k8s/helm/nexent/charts/nexent-common/files/init.sql` (K8s fresh deploy) + +**Version**: Tracked in `backend/consts/const.py` as `APP_VERSION` + +--- + +## Testing Conventions + +- pytest only (no unittest) +- Mock at import site with fully-qualified path: + ```python + mocker.patch("backend.services.agent_service.AgentService.run", return_value={...}) + ``` +- Async tests: `@pytest.mark.asyncio` +- Test structure: `test/backend/` and `test/sdk/` + +--- + +## Code Style + +- English-only comments and docstrings (enforced by `.cursor/rules/english_comments.mdc`) +- Import order: stdlib → third-party → project +- Line length: 119 (sdk ruff config) + +--- + +## Key Files + +| File | Purpose | +|------|---------| +| `backend/consts/const.py` | All env var definitions, APP_VERSION | +| `backend/consts/exceptions.py` | Domain exceptions (AgentRunException, LimitExceededError, etc.) | +| `docker/init.sql` | Database schema for Docker Compose | +| `k8s/helm/.../init.sql` | Database schema for Kubernetes | +| `test/run_all_test.py` | Test runner with coverage | + +--- + +## Reference Files + +Existing instruction files with detailed rules: +- `CLAUDE.md` - Backend architecture, env var management, app/service layer rules +- `.cursor/rules/environment_variable.mdc` - Env var centralization +- `.cursor/rules/pytest_unit_test_rules.mdc` - Testing patterns +- `.cursor/rules/english_comments.mdc` - Comment language enforcement \ No newline at end of file diff --git a/README.md b/README.md index 7983e6c6c..236f603aa 100644 --- a/README.md +++ b/README.md @@ -46,13 +46,17 @@ Quick and straightforward for most users. Prerequisites: Docker 24+ and Docker C ```bash git clone https://github.com/ModelEngine-Group/nexent.git -cd nexent/docker -bash deploy.sh +cd nexent +bash deploy.sh docker ``` -The Docker and Kubernetes deploy scripts share the same deployment configuration model. Interactive runs show Bash TUI menus for component selection, port policy, and image source. `infrastructure` is required; `application` is selected by default but can be disabled. Use `b`/Backspace to return to the previous TUI step and `q` to quit. Non-interactive runs can pass the same choices with `--components`, `--port-policy development|production`, and `--image-source general|mainland|local-latest`. Successful deployments save non-sensitive choices to each deploy directory's `deploy.options` for reuse on the next run. +The root `deploy.sh` only forwards to the target deploy script; the native Docker implementation is `bash deploy/docker/deploy.sh`. The Docker and Kubernetes deploy scripts share the same deployment configuration model. Interactive runs show Bash TUI menus for component selection, port policy, and image source. `infrastructure` is required; `application`, `data-process`, and `supabase` are selected by default and can be disabled when you want a smaller deployment. Use `b`/Backspace to return to the previous TUI step and `q` to quit. Non-interactive runs can pass the same choices with `--version`, `--components`, `--port-policy development|production`, and `--image-source general|mainland|local-latest`. Successful deployments save non-sensitive choices to each deploy directory's `deploy.options` for reuse on the next run. -Docker uninstall is handled by `bash uninstall.sh`. It can preserve or delete data volumes: run it interactively, pass `--delete-volumes true|false`, or use `bash uninstall.sh delete-all` to remove containers and persistent data. +Docker and Kubernetes both use the project root `.env` as the runtime configuration file. Existing `.env` is kept as-is. If it does not exist, the deploy scripts first reuse an existing `docker/.env`, then fall back to `.env.example` or `docker/.env.example`. + +Docker uninstall is handled by `bash uninstall.sh docker`. It can preserve or delete data volumes: run it interactively, pass `--delete-volumes true|false`, or use `bash uninstall.sh docker delete-all` to remove containers and persistent data. + +Offline image packages can be built with `bash deploy/offline/build_offline_package.sh --target docker --compress true`. The package includes image tar files, `load-images.sh`, root deploy/uninstall entrypoints, deployment scripts, SQL files, `manifest.yaml`, and `checksums.txt`; deploy it with `bash deploy.sh --load-images docker ...` on the target host. For detailed deployment instructions, see [Docker Installation](https://modelengine-group.github.io/nexent/en/quick-start/installation.html). @@ -62,11 +66,15 @@ Ideal for enterprise scenarios requiring high availability and elastic scaling. ```bash git clone https://github.com/ModelEngine-Group/nexent.git -cd nexent/k8s/helm -./deploy.sh +cd nexent +bash deploy.sh k8s ``` -Kubernetes uninstall is handled by `bash uninstall.sh`. It removes the Helm release first, then can optionally delete the namespace and local hostPath data. Use `--delete-namespace true|false`, `--delete-local-data true|false`, or `bash uninstall.sh delete-all`; pass `--keep-local-data` with `delete-all` to preserve local volume contents. +The native Kubernetes implementation is `bash deploy/k8s/deploy.sh`. It reads the same project root `.env` as Docker and renders explicit values into Helm ConfigMap and Secret overrides. Use `--persistence-mode local|dynamic|existing`, `--storage-class`/`--sc`, `--local-path`, `--local-node-name`, and `--existing-claim-prefix` to control PVC behavior. Local mode renders `hostPath` PVs and does not require node affinity. + +Kubernetes uninstall is handled by `bash uninstall.sh k8s`. It removes the Helm release first, then can optionally delete the namespace and local PV data. Use `--delete-namespace true|false`, `--delete-local-data true|false`, or `bash uninstall.sh k8s delete-all`; pass `--keep-local-data` with `delete-all` to preserve local volume contents. + +Kubernetes offline packages use the same builder with `--target k8s` or `--target all`. Run `load-images.sh` on every cluster node that needs the images, or push the loaded images to an internal registry before deploying with the same version and image-source options used during packaging. For detailed deployment instructions, see [Kubernetes Installation](https://modelengine-group.github.io/nexent/en/quick-start/kubernetes-installation.html). diff --git a/README_CN.md b/README_CN.md index 032776418..5d27fa4aa 100644 --- a/README_CN.md +++ b/README_CN.md @@ -46,11 +46,18 @@ Nexent 是一个基于 **Harness Engineering** 原则打造的零代码智能体 ```bash git clone https://github.com/ModelEngine-Group/nexent.git -cd nexent/docker -cp .env.example .env -bash deploy.sh +cd nexent +bash deploy.sh docker ``` +根目录 `deploy.sh` 只负责转发到目标部署脚本；Docker 真实实现为 `bash deploy/docker/deploy.sh`。Docker 和 Kubernetes 使用同一套部署配置模型；交互式运行会通过 Bash TUI 选择组件、端口策略和镜像源。`infrastructure` 必选，`application`、`data-process`、`supabase` 默认选中，也可以取消以部署更小的组合。非交互部署可传入 `--version`、`--components`、`--port-policy development|production`、`--image-source general|mainland|local-latest`。 + +Docker 与 Kubernetes 统一使用项目根目录 `.env` 作为运行配置文件；已有 `.env` 会原样保留。如果根目录 `.env` 不存在，部署脚本会优先复用已有的 `docker/.env`，再回退到 `.env.example` 或 `docker/.env.example`。 + +Docker 卸载入口为 `bash uninstall.sh docker`，默认交互确认是否删除持久化数据；也可以通过 `--delete-volumes true|false` 控制，或使用 `bash uninstall.sh docker delete-all` 同时删除容器和持久化数据。 + +离线镜像包可通过 `bash deploy/offline/build_offline_package.sh --target docker --compress true` 构建。包内包含镜像 tar、`load-images.sh`、根目录部署/卸载入口、部署脚本、SQL 文件、`manifest.yaml` 和 `checksums.txt`；在目标机器上使用 `bash deploy.sh --load-images docker ...` 加载镜像并部署。 + 详细部署指南请参考 [Docker 安装部署](https://modelengine-group.github.io/nexent/zh/quick-start/installation.html)。 ### Kubernetes 部署（适合企业级生产环境） @@ -59,10 +66,16 @@ bash deploy.sh ```bash git clone https://github.com/ModelEngine-Group/nexent.git -cd nexent/k8s/helm -./deploy-helm.sh apply +cd nexent +bash deploy.sh k8s ``` +Kubernetes 真实实现为 `bash deploy/k8s/deploy.sh`。它会读取同一个根目录 `.env`，并显式渲染为 Helm ConfigMap 和 Secret 覆盖值。PVC 可通过 `--persistence-mode local|dynamic|existing`、`--storage-class`/`--sc`、`--local-path`、`--local-node-name`、`--existing-claim-prefix` 控制。local 模式会渲染 `hostPath` PV，不再需要 nodeAffinity。 + +根目录卸载入口为 `bash uninstall.sh docker ...` 或 `bash uninstall.sh k8s ...`，具体实现仍分别在 `deploy/docker/uninstall.sh` 和 `deploy/k8s/uninstall.sh`。 + +Kubernetes 离线包使用同一个构建脚本，传入 `--target k8s` 或 `--target all`。部署前需要在每个需要运行 Pod 的节点上执行 `load-images.sh`，或将镜像推送到集群可访问的内部镜像仓库，再使用与打包时一致的版本和镜像源参数部署。 + 详细部署指南请参考 [Kubernetes 安装部署](https://modelengine-group.github.io/nexent/zh/quick-start/kubernetes-installation.html)。 # ✨ 核心特性 diff --git a/VERSION b/VERSION new file mode 100644 index 000000000..7fe52d367 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +v2.2.1 diff --git a/backend/agents/agent_run_manager.py b/backend/agents/agent_run_manager.py index 83a05aa2a..eca8c2fa4 100644 --- a/backend/agents/agent_run_manager.py +++ b/backend/agents/agent_run_manager.py @@ -1,11 +1,13 @@ -import logging -import threading -from typing import Dict, Union - -from nexent.core.agents.agent_model import AgentRunInfo -from nexent.core.agents.agent_context import ContextManager, ContextManagerConfig - -logger = logging.getLogger("agent_run_manager") +import logging +import threading +from typing import TYPE_CHECKING, Any, Dict, Union + +from nexent.core.agents.agent_model import AgentRunInfo + +if TYPE_CHECKING: + from nexent.core.agents.agent_context import ContextManager, ContextManagerConfig + +logger = logging.getLogger("agent_run_manager") class AgentRunManager: @@ -22,10 +24,10 @@ def __new__(cls): def __init__(self): if not self._initialized: - # user_id:conversation_id -> agent_run_info - self.agent_runs: Dict[str, AgentRunInfo] = {} - # conversation_id -> ContextManager (conversation-level lifetime) - self._conversation_context_managers: Dict[str, ContextManager] = {} + # user_id:conversation_id -> agent_run_info + self.agent_runs: Dict[str, AgentRunInfo] = {} + # conversation_id -> ContextManager (conversation-level lifetime) + self._conversation_context_managers: Dict[str, Any] = {} # conversation_id -> active run count for safe cleanup self._conversation_run_counts: Dict[str, int] = {} self._initialized = True @@ -76,13 +78,15 @@ def stop_agent_run(self, conversation_id: Union[int, str], user_id: str) -> bool return False def get_or_create_context_manager( - self, - conversation_id: Union[int, str], - config: ContextManagerConfig, - max_steps: int - ) -> ContextManager: - """Get or create a conversation-level ContextManager instance.""" - conv_key = str(conversation_id) + self, + conversation_id: Union[int, str], + config: "ContextManagerConfig", + max_steps: int + ) -> "ContextManager": + """Get or create a conversation-level ContextManager instance.""" + from nexent.core.agents.agent_context import ContextManager + + conv_key = str(conversation_id) with self._lock: cm = self._conversation_context_managers.get(conv_key) if cm is None: diff --git a/backend/agents/create_agent_info.py b/backend/agents/create_agent_info.py index 7e3b42e28..220a66914 100644 --- a/backend/agents/create_agent_info.py +++ b/backend/agents/create_agent_info.py @@ -7,9 +7,23 @@ from jinja2 import Template, StrictUndefined from nexent.core.utils.observer import MessageObserver from nexent.core.agents.agent_model import AgentRunInfo, ModelConfig, AgentConfig, ToolConfig, ExternalA2AAgentConfig, AgentHistory, AgentVerificationConfig -from nexent.core.agents.agent_context import ContextManagerConfig +from nexent.core.agents.summary_config import ContextManagerConfig +from nexent.core.models.prompt_cache import resolve_prompt_cache_profile +from nexent.core.models.capacity_resolver import ( + ModelCapacitySnapshot, + ProviderCapabilityUnknown, + ResolverError, + resolve_capacity, +) +from nexent.core.models.capacity_budget import ( + RequestBudgetOverrides, + SafeInputBudgetCalculator, + UncertaintyReserveBasisUnknown, +) from nexent.memory.memory_service import search_memory_in_levels +from consts.capability_profiles import CATALOG as CAPABILITY_CATALOG + from services.file_management_service import get_llm_model, validate_urls_access from services.vectordatabase_service import ( ElasticSearchService, @@ -44,6 +58,229 @@ logger.setLevel(logging.DEBUG) +# Safe fallback for context-manager token_threshold when no capacity is known. +# Used only when the resolver fails (uncataloged model with no operator-supplied +# hard capacity). Sized to cover the typical 32K-context band shared by the +# majority of production LLMs (GPT-3.5 16K, GLM-4 32K, Qwen2 32K, Llama 3 +# 32K, etc.). Larger windows benefit only by skipping a few extra +# compressions; smaller ones surface as a clear provider token-overflow +# error at request time rather than silent truncation. Will be removed +# once enforcement phase requires snapshots end to end. +_TOKEN_THRESHOLD_LEGACY_FALLBACK = 32768 + +_OPERATOR_OVERRIDE_FIELDS = ( + "context_window_tokens", + "max_input_tokens", + "max_output_tokens", + "default_output_reserve_tokens", + "tokenizer_family", +) + +# Per-process dedup for the "model has no capacity configured" warning. +# Without this, every agent run logs the same line, drowning real signal. +# Keyed by model_id; cleared only on process restart. +# Guarded by a lock because the check-then-add window is not atomic on its +# own: two threads can both pass the `in` check before either calls `add`, +# leading to duplicate WARNING lines defeating the per-process dedup. +_CAPACITY_WARNING_EMITTED: set = set() +_CAPACITY_WARNING_LOCK = threading.Lock() + + +def _operator_overrides_from_model_info(model_info: Optional[dict]) -> dict: + """Extract the W1 operator-override fields from a model_record_t row.""" + if not isinstance(model_info, dict): + return {} + overrides = {} + for field in _OPERATOR_OVERRIDE_FIELDS: + value = model_info.get(field) + if value is not None: + overrides[field] = value + return overrides + + +def _dominant_capacity_source(field_sources: dict) -> Optional[str]: + values = [value for value in field_sources.values() if value] + if not values: + return None + for preferred in ("operator", "profile", "provider_candidate", "legacy", "unknown"): + if preferred in values: + return preferred + return values[0] + + +def _capacity_snapshot_for_monitoring(snapshot: Any) -> dict: + data = snapshot.model_dump() if hasattr(snapshot, "model_dump") else dict(snapshot) + return { + "provider": data.get("provider"), + "model_name": data.get("model_name"), + "context_window_tokens": data.get("context_window_tokens"), + "default_output_reserve_tokens": data.get("default_output_reserve_tokens"), + "capability_profile_version": data.get("capability_profile_version"), + "capacity_source": _dominant_capacity_source(data.get("field_sources") or {}), + "requested_output_tokens": data.get("requested_output_tokens"), + "provider_input_limit_tokens": data.get("provider_input_limit_tokens"), + "tokenizer_family": data.get("tokenizer_family"), + "counting_mode": data.get("counting_mode"), + "unknown_capabilities": data.get("unknown_capabilities") or [], + "capacity_fingerprint": data.get("fingerprint"), + } + + +def _safe_input_budget_for_monitoring(snapshot: Any) -> dict: + return snapshot.model_dump() if hasattr(snapshot, "model_dump") else dict(snapshot) + + +def _resolve_safe_input_budget( + *, + capacity_snapshot: Optional[ModelCapacitySnapshot], + tenant_id: str, + agent_requested_output_tokens: Optional[int], + request_requested_output_tokens: Optional[int], +) -> Optional[dict]: + """Resolve the W2 budget snapshot before context assembly begins.""" + if capacity_snapshot is None: + return None + + request_overrides = None + if request_requested_output_tokens is not None: + request_overrides = RequestBudgetOverrides( + requested_output_tokens=request_requested_output_tokens, + ) + + output_reserve_source = ( + "agent" if agent_requested_output_tokens is not None else "model_default" + ) + try: + snapshot = SafeInputBudgetCalculator().calculate_safe_input_budget( + capacity_snapshot=capacity_snapshot, + reserve_policy=tenant_config_manager.get_capacity_reserve_policy(tenant_id), + request_overrides=request_overrides, + requested_output_tokens=agent_requested_output_tokens, + output_reserve_source=output_reserve_source, + ) + except UncertaintyReserveBasisUnknown as exc: + # W2 uncertainty reserve needs context_window_tokens as the 10% basis. + # Falls through here when a model row has max_input_tokens set but + # context_window_tokens is NULL — possible for rows imported before + # W11 V1 save-time defaults landed, or for rows written directly via + # SQL/legacy import. Degrade to the same "no W2 snapshot" branch the + # caller already handles (falls back to W1 input_budget). + logger.warning( + "W2 safe input budget unavailable (tenant_id=%s model=%s): %s - " + "falling back to W1 input_budget. Fill context_window_tokens on the " + "model record to enable W2 enforcement.", + tenant_id, + capacity_snapshot.model_name, + exc, + ) + return None + logger.info( + "W2 safe input budget resolved: tenant_id=%s model=%s requested_output_tokens=%s " + "soft_input_budget_tokens=%s hard_input_budget_tokens=%s fingerprint=%s warnings=%s", + tenant_id, + snapshot.model_name, + snapshot.requested_output_tokens, + snapshot.soft_input_budget_tokens, + snapshot.hard_input_budget_tokens, + snapshot.fingerprint, + list(snapshot.warnings), + ) + return _safe_input_budget_for_monitoring(snapshot) + + +def _resolve_input_budget( + model_info: Optional[dict], +) -> tuple[int, Optional[dict], Optional[ModelCapacitySnapshot]]: + """Resolve the context-manager input budget for a model_record_t row. + + Calls ModelCapacityResolver with the catalog + operator overrides. Returns + snapshot.provider_input_limit_tokens and monitoring fields on success. + Falls back to _TOKEN_THRESHOLD_LEGACY_FALLBACK with no snapshot when + capacity is unknown — this is the migration-window behavior before all + model rows are backfilled. + """ + if not isinstance(model_info, dict): + return _TOKEN_THRESHOLD_LEGACY_FALLBACK, None, None + provider_raw = model_info.get("model_factory") + provider = provider_raw.lower().strip() if isinstance(provider_raw, str) else "" + model_id = model_info.get("model_name") or "" + provider_missing_detail = None + if not provider: + provider_missing_detail = ( + "model_factory/provider is missing; capacity catalog matching is disabled" + ) + try: + snapshot = resolve_capacity( + model_id=model_id, + provider=provider, + operator_overrides=_operator_overrides_from_model_info(model_info), + capability_profiles=CAPABILITY_CATALOG, + ) + logger.debug( + "Capacity resolved for (%s, %s): input_limit=%s source=%s profile=%s fingerprint=%s", + provider, model_id, + snapshot.provider_input_limit_tokens, + dict(snapshot.field_sources), + snapshot.capability_profile_version, + snapshot.fingerprint, + ) + return ( + snapshot.provider_input_limit_tokens, + _capacity_snapshot_for_monitoring(snapshot), + snapshot, + ) + except ProviderCapabilityUnknown: + _warn_missing_capacity_once( + model_info, provider, model_id, detail=provider_missing_detail, + ) + return _TOKEN_THRESHOLD_LEGACY_FALLBACK, None, None + except ResolverError as exc: + _warn_missing_capacity_once( + model_info, provider, model_id, detail=str(exc), + ) + return _TOKEN_THRESHOLD_LEGACY_FALLBACK, None, None + + +def _warn_missing_capacity_once( + model_info: Optional[dict], + provider: str, + model_id_str: str, + detail: Optional[str] = None, +) -> None: + """Log one WARNING per process per model when capacity is not configured. + + Plain-English message aimed at operators reading backend logs. Tells + them what is disabled, which model is affected, and how to fix it + through the existing UI. + """ + db_model_id = ( + model_info.get("model_id") if isinstance(model_info, dict) else None + ) + dedup_key = db_model_id if db_model_id is not None else f"{provider}/{model_id_str}" + # Test-and-set inside the lock so concurrent first-time callers don't + # both make it past the membership check. Logging happens outside the + # lock to avoid serialising I/O across all warning paths. + with _CAPACITY_WARNING_LOCK: + if dedup_key in _CAPACITY_WARNING_EMITTED: + return + _CAPACITY_WARNING_EMITTED.add(dedup_key) + + reason = ( + f"resolver error: {detail}" + if detail + else "no context_window_tokens or max_output_tokens configured" + ) + logger.warning( + "Output token cap and budget consistency check are not enforced for " + "model '%s' (model_id=%s, provider=%s) because %s. " + "To enable enforcement, open the Nexent model management UI, edit " + "this model, and fill in 'Context window tokens' and 'Max output " + "tokens'. Falling back to a default context threshold of %s tokens.", + model_id_str, db_model_id, provider, reason, + _TOKEN_THRESHOLD_LEGACY_FALLBACK, + ) + + def _normalize_tool_params_request(tool_params: Optional[ToolParamsRequest | Dict[str, Any]]) -> ToolParamsRequest: """Normalize request-scoped tool parameter overrides into a ToolParamsRequest.""" if tool_params is None: @@ -153,7 +390,7 @@ def _get_skills_for_template( for s in enabled_skills ] except Exception as e: - logger.warning(f"Failed to get skills for template: {e}") + logger.error(f"Failed to get skills for agent {agent_id} (tenant={tenant_id}, version={version_no}): {e}", exc_info=True) return [] @@ -336,10 +573,24 @@ async def create_model_config_list(tenant_id): ssl_verify=record.get("ssl_verify", True), model_factory=record.get("model_factory"), timeout_seconds=record.get("timeout_seconds"), - concurrency_limit=record.get("concurrency_limit"))) + concurrency_limit=record.get("concurrency_limit"), + prompt_cache=resolve_prompt_cache_profile( + record.get("model_factory")), + # W1 step 6: pass capacity columns through so SDK can + # honor operator-configured values end to end. + max_output_tokens=record.get("max_output_tokens"), + max_tokens=record.get("max_tokens"), + context_window_tokens=record.get("context_window_tokens"), + max_input_tokens=record.get("max_input_tokens"), + default_output_reserve_tokens=record.get("default_output_reserve_tokens"), + tokenizer_family=record.get("tokenizer_family"), + capacity_source=record.get("capacity_source"), + capability_profile_version=record.get("capability_profile_version"))) # fit for old version, main_model and sub_model use default model main_model_config = tenant_config_manager.get_model_config( key=MODEL_CONFIG_MAPPING["llm"], tenant_id=tenant_id) + main_prompt_cache = resolve_prompt_cache_profile( + main_model_config.get("model_factory")) model_list.append( ModelConfig(cite_name="main_model", api_key=main_model_config.get("api_key", ""), @@ -349,7 +600,8 @@ async def create_model_config_list(tenant_id): ssl_verify=main_model_config.get("ssl_verify", True), model_factory=main_model_config.get("model_factory"), timeout_seconds=main_model_config.get("timeout_seconds"), - concurrency_limit=main_model_config.get("concurrency_limit"))) + concurrency_limit=main_model_config.get("concurrency_limit"), + prompt_cache=main_prompt_cache)) model_list.append( ModelConfig(cite_name="sub_model", api_key=main_model_config.get("api_key", ""), @@ -359,7 +611,8 @@ async def create_model_config_list(tenant_id): ssl_verify=main_model_config.get("ssl_verify", True), model_factory=main_model_config.get("model_factory"), timeout_seconds=main_model_config.get("timeout_seconds"), - concurrency_limit=main_model_config.get("concurrency_limit"))) + concurrency_limit=main_model_config.get("concurrency_limit"), + prompt_cache=main_prompt_cache)) return model_list @@ -373,6 +626,7 @@ async def create_agent_config( allow_memory_search: bool = True, version_no: int = 0, override_model_id: int | None = None, + request_requested_output_tokens: int | None = None, tool_params: Optional[ToolParamsRequest | Dict[str, Any]] = None, ): normalized_tool_params = _normalize_tool_params_request(tool_params) @@ -531,6 +785,7 @@ async def create_agent_config( # Build knowledge base summary knowledge_base_summary = "" + kb_ids = [] try: for tool in tool_list: if "KnowledgeBaseSearchTool" == tool.class_name: @@ -545,6 +800,7 @@ async def create_agent_config( message = ElasticSearchService().get_summary(index_name=index_name) summary = message.get("summary", "") knowledge_base_summary += f"**{display_name}**: {summary}\n\n" + kb_ids.append(index_name) except Exception as e: logger.warning( f"Failed to get summary for knowledge base {index_name}: {e}") @@ -555,7 +811,11 @@ async def create_agent_config( except Exception as e: logger.error(f"Failed to build knowledge base summary: {e}") - # Assemble system_prompt + # Select the context path once. Managed assembly receives raw components + # and must never consume a Jinja-rendered legacy prompt. + enable_context_manager = agent_info.get("enable_context_manager", False) + + # Assemble legacy system_prompt only for the isolated fallback path. # Get skills list for prompt template skills = _get_skills_for_template(agent_id, tenant_id, version_no) @@ -575,24 +835,56 @@ async def create_agent_config( "knowledge_base_summary": knowledge_base_summary, "user_id": user_id, } - system_prompt = Template(prompt_template["system_prompt"], undefined=StrictUndefined).render(render_kwargs) + system_prompt = "" + if not enable_context_manager: + system_prompt = Template( + prompt_template["system_prompt"], undefined=StrictUndefined + ).render(render_kwargs) model_id_to_use = override_model_id if override_model_id else agent_info.get("model_id") - model_max_tokens = 10000 + model_info = None if model_id_to_use is not None: model_info = get_model_by_model_id(model_id_to_use, tenant_id=tenant_id) model_name = model_info["display_name"] if model_info is not None else "main_model" - if model_info is not None and model_info.get("max_tokens"): - model_max_tokens = model_info["max_tokens"] + # W1 step 6: derive input budget via ModelCapacityResolver instead of + # treating model_info["max_tokens"] (a deprecated output cap) as a + # context threshold. Falls back to a safe constant when capacity is + # unknown during the migration window. + input_budget, capacity_snapshot, resolved_capacity_snapshot = ( + _resolve_input_budget(model_info) + ) else: model_name = "main_model" + input_budget = _TOKEN_THRESHOLD_LEGACY_FALLBACK + capacity_snapshot = None + resolved_capacity_snapshot = None + + requested_output_tokens = agent_info.get("requested_output_tokens") + safe_input_budget_snapshot = _resolve_safe_input_budget( + capacity_snapshot=resolved_capacity_snapshot, + tenant_id=tenant_id, + agent_requested_output_tokens=requested_output_tokens, + request_requested_output_tokens=request_requested_output_tokens, + ) + if safe_input_budget_snapshot is not None: + soft_input_budget_tokens = safe_input_budget_snapshot["soft_input_budget_tokens"] + hard_input_budget_tokens = safe_input_budget_snapshot["hard_input_budget_tokens"] + context_token_threshold = soft_input_budget_tokens + else: + soft_input_budget_tokens = 0 + hard_input_budget_tokens = 0 + context_token_threshold = input_budget - # Use agent-level setting for context management, default to False. - # When ContextManager is disabled, do not attach context_components because - # downstream runtime may prefer component-based prompt assembly over the - # rendered system_prompt, causing the actual model input to diverge from the - # template output. - enable_context_manager = agent_info.get("enable_context_manager", False) + logger.info( + "Agent main LLM: agent_id=%s, model_id=%s, display_name=%s, model_name=%s", + agent_id, + model_id_to_use, + model_info.get("display_name") if model_info else model_name, + model_info.get("model_name") if model_info else model_name, + ) + + # Managed context assembly starts from raw sources. No legacy rendered + # prompt is supplied on this path. context_components = [] if enable_context_manager: context_components = build_context_components( @@ -611,10 +903,19 @@ async def create_agent_config( memory_list=memory_list, memory_search_query=last_user_query, knowledge_base_summary=knowledge_base_summary, + kb_ids=kb_ids, + ) + + logger.info( + f"Agent {agent_id} context assembly: " + f"skills_count={len(skills)}, " + f"components={[f'{type(c).__name__}(type={c.component_type},priority={c.priority})' for c in context_components]}" ) cm_config = ContextManagerConfig( enabled=enable_context_manager, - token_threshold=model_max_tokens, + token_threshold=context_token_threshold, + soft_input_budget_tokens=soft_input_budget_tokens, + hard_input_budget_tokens=hard_input_budget_tokens, ) agent_config = AgentConfig( name="undefined" if agent_info["name"] is None else agent_info["name"], @@ -627,12 +928,15 @@ async def create_agent_config( ), tools=tool_list + _get_skill_script_tools(agent_id, tenant_id, version_no), max_steps=agent_info.get("max_steps", 15), + requested_output_tokens=requested_output_tokens, model_name=model_name, provide_run_summary=agent_info.get("provide_run_summary", False), managed_agents=managed_agents, external_a2a_agents=external_a2a_agents, context_manager_config=cm_config, context_components=context_components, + capacity_snapshot=capacity_snapshot, + safe_input_budget_snapshot=safe_input_budget_snapshot, verification_config=AgentVerificationConfig.model_validate(agent_info.get("verification_config") or {}), ) return agent_config @@ -759,22 +1063,25 @@ async def create_tool_config_list( "rerank_model": rerank_model, } elif tool_config.class_name == "AnalyzeTextFileTool": + selected_model_id = param_dict.get("selected_model_id") tool_config.metadata = { - "llm_model": get_llm_model(tenant_id=tenant_id), + "llm_model": get_llm_model(tenant_id=tenant_id, model_id=selected_model_id), "storage_client": minio_client, "data_process_service_url": DATA_PROCESS_SERVICE, "validate_url_access": lambda urls: validate_urls_access(urls, user_id) } elif tool_config.class_name == "AnalyzeImageTool": + selected_model_id = param_dict.get("selected_model_id") tool_config.metadata = { # get_vlm_model reads the first multimodal slot, now shown as image understanding. - "vlm_model": get_vlm_model(tenant_id=tenant_id), + "vlm_model": get_vlm_model(tenant_id=tenant_id, model_id=selected_model_id), "storage_client": minio_client, "validate_url_access": lambda urls: validate_urls_access(urls, user_id) } elif tool_config.class_name in ["AnalyzeAudioTool", "AnalyzeVideoTool"]: + selected_model_id = param_dict.get("selected_model_id") tool_config.metadata = { - "vlm_model": get_video_understanding_model(tenant_id=tenant_id), + "vlm_model": get_video_understanding_model(tenant_id=tenant_id, model_id=selected_model_id), "storage_client": minio_client, "validate_url_access": lambda urls: validate_urls_access(urls, user_id) } @@ -1042,6 +1349,7 @@ async def create_agent_run_info( is_debug: bool = False, override_version_no: int | None = None, override_model_id: int | None = None, + requested_output_tokens: int | None = None, tool_params: Optional[ToolParamsRequest | Dict[str, Any]] = None, ): # Determine which version_no to use based on is_debug flag @@ -1074,6 +1382,8 @@ async def create_agent_run_info( } if override_model_id is not None: create_config_kwargs["override_model_id"] = override_model_id + if requested_output_tokens is not None: + create_config_kwargs["request_requested_output_tokens"] = requested_output_tokens agent_config = await create_agent_config(**create_config_kwargs, tool_params=tool_params) @@ -1129,6 +1439,12 @@ async def create_agent_run_info( agent_config=agent_config, mcp_host=mcp_host, history=converted_history, - stop_event=threading.Event() + stop_event=threading.Event(), + capacity_snapshot=getattr(agent_config, "capacity_snapshot", None), + safe_input_budget_snapshot=getattr( + agent_config, + "safe_input_budget_snapshot", + None, + ), ) return agent_run_info diff --git a/backend/apps/agent_repository_app.py b/backend/apps/agent_repository_app.py index e9da2fde0..f538d5bf7 100644 --- a/backend/apps/agent_repository_app.py +++ b/backend/apps/agent_repository_app.py @@ -1,4 +1,3 @@ -import logging from http import HTTPStatus from typing import Optional @@ -6,38 +5,96 @@ from starlette.responses import JSONResponse from consts.exceptions import SkillDuplicateError, UnauthorizedError +from consts.model import AgentRepositoryListingCreateRequest from services.agent_repository_service import ( create_agent_repository_listing_impl, + get_agent_repository_listing_detail_impl, import_agent_from_repository_impl, list_agent_repository_listings_impl, + list_my_editable_agents_impl, update_agent_repository_status_impl, ) from utils.auth_utils import get_current_user_id agent_repository_router = APIRouter(prefix="/repository/agent") -logger = logging.getLogger("agent_repository_app") @agent_repository_router.get("") async def list_agent_repository_listings_api( status: Optional[str] = Query(None, description="Filter by listing status"), + agent_id: Optional[int] = Query(None, description="Filter by source agent ID"), + deduplicate_by_agent_id: Optional[bool] = Query( + None, + description="Whether to return one listing per agent", + ), + category_id: Optional[int] = Query( + None, + description="Filter by marketplace category ID", + ), authorization: str = Header(None), ): """List all marketplace repository listings with optional status filter.""" try: - get_current_user_id(authorization) - result = list_agent_repository_listings_impl(status=status) + _, tenant_id = get_current_user_id(authorization) + should_deduplicate = ( + agent_id is None + if deduplicate_by_agent_id is None + else deduplicate_by_agent_id + ) + result = list_agent_repository_listings_impl( + tenant_id, + status=status, + agent_id=agent_id, + deduplicate_by_agent_id=should_deduplicate, + category_id=category_id, + ) return JSONResponse(status_code=HTTPStatus.OK, content=result) except UnauthorizedError as e: raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) except ValueError as e: raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) - except Exception as e: - logger.error(f"List agent repository listings error: {str(e)}") - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="List agent repository listings error.", + + +@agent_repository_router.get("/mine") +async def list_my_editable_agents_api( + ownership: Optional[str] = Query( + "all", + description="Filter by ownership: all / created / others", + ), + authorization: str = Header(None), +): + """List editable draft agents for the current user with repository listing info.""" + try: + user_id, tenant_id = get_current_user_id(authorization) + result = list_my_editable_agents_impl( + tenant_id=tenant_id, + user_id=user_id, + ownership=ownership or "all", + ) + return JSONResponse(status_code=HTTPStatus.OK, content=result) + except UnauthorizedError as e: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) + + +@agent_repository_router.get("/{agent_repository_id}") +async def get_agent_repository_listing_detail_api( + agent_repository_id: int, + authorization: str = Header(None), +): + """Get detailed marketplace repository listing by primary key.""" + try: + _, tenant_id = get_current_user_id(authorization) + result = get_agent_repository_listing_detail_impl( + agent_repository_id, + tenant_id, ) + return JSONResponse(status_code=HTTPStatus.OK, content=result) + except UnauthorizedError as e: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) @agent_repository_router.patch("/{agent_repository_id}/status") @@ -47,59 +104,51 @@ async def update_agent_repository_status_api( ..., embed=True, description=( - "New status: NOT_SHARED (未共享) / PENDING_REVIEW (待审核) / " - "REJECTED (审核驳回) / SHARED (已共享)" + "New status: not_shared (未共享) / pending_review (待审核) / " + "rejected (审核驳回) / shared (已共享)" ), ), authorization: str = Header(None), ): """Update marketplace repository listing status (share, unshare, approve, reject).""" try: - user_id, _ = get_current_user_id(authorization) + user_id, tenant_id = get_current_user_id(authorization) result = update_agent_repository_status_impl( agent_repository_id=agent_repository_id, status=status, user_id=user_id, + tenant_id=tenant_id, ) return JSONResponse(status_code=HTTPStatus.OK, content=result) except UnauthorizedError as e: raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) except ValueError as e: raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) - except Exception as e: - logger.error(f"Update agent repository status error: {str(e)}") - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Update agent repository status error.", - ) @agent_repository_router.post("/{agent_id}/versions/{version_no}") async def create_agent_repository_listing_api( agent_id: int, version_no: int, + payload: Optional[AgentRepositoryListingCreateRequest] = Body(None), authorization: str = Header(None), ): """Create or update a marketplace repository listing from an agent version snapshot.""" try: user_id, tenant_id = get_current_user_id(authorization) + card_fields = payload.model_dump(exclude_none=True) if payload else None result = await create_agent_repository_listing_impl( agent_id=agent_id, tenant_id=tenant_id, user_id=user_id, version_no=version_no, + card_fields=card_fields, ) return JSONResponse(status_code=HTTPStatus.OK, content=result) except UnauthorizedError as e: raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) except ValueError as e: raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) - except Exception as e: - logger.error(f"Create agent repository listing error: {str(e)}") - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Create agent repository listing error.", - ) @agent_repository_router.post("/{agent_repository_id}/import") @@ -109,8 +158,10 @@ async def import_agent_from_repository_api( ): """Import an agent tree from a marketplace repository listing into the current tenant.""" try: + _, tenant_id = get_current_user_id(authorization) await import_agent_from_repository_impl( agent_repository_id=agent_repository_id, + tenant_id=tenant_id, authorization=authorization, ) return JSONResponse(status_code=HTTPStatus.OK, content={}) @@ -126,9 +177,3 @@ async def import_agent_from_repository_api( ) except ValueError as e: raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) - except Exception as e: - logger.error(f"Import agent from repository error: {str(e)}") - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Import agent from repository error.", - ) diff --git a/backend/apps/aidp_app.py b/backend/apps/aidp_app.py new file mode 100644 index 000000000..49f7006f9 --- /dev/null +++ b/backend/apps/aidp_app.py @@ -0,0 +1,72 @@ +""" +AIDP App Layer +FastAPI endpoints for AIDP knowledge base list proxy. +""" +import logging +from http import HTTPStatus +from typing import Annotated + +from fastapi import APIRouter, Query +from fastapi.responses import JSONResponse + +from consts.error_code import ErrorCode +from consts.exceptions import AppException +from services.aidp_service import ( + fetch_aidp_knowledge_bases_impl, + fetch_all_aidp_knowledge_bases_impl, +) + +router = APIRouter(prefix="/aidp") +logger = logging.getLogger("aidp_app") + + +@router.get("/knowledge-bases") +async def fetch_aidp_knowledge_bases_api( + server_url: Annotated[str, Query(description="AIDP API server URL")], + api_key: Annotated[str, Query(description="AIDP API key")], + page: Annotated[int, Query(ge=1, description="Page number starting from 1")] = 1, + page_size: Annotated[int, Query(ge=1, le=100, description="Page size from 1 to 100")] = 10, +) -> JSONResponse: + """Fetch a single page of knowledge bases from the external AIDP API.""" + try: + result = fetch_aidp_knowledge_bases_impl( + server_url=server_url, + api_key=api_key, + page=page, + page_size=page_size, + ) + return JSONResponse(status_code=HTTPStatus.OK, content=result) + except AppException: + raise + except Exception as e: + logger.exception("Failed to fetch AIDP knowledge bases: %s", e) + raise AppException( + ErrorCode.AIDP_SERVICE_ERROR, + f"Failed to fetch AIDP knowledge bases: {str(e)}", + ) + + +@router.get("/knowledge-bases-all") +async def fetch_all_aidp_knowledge_bases_api( + server_url: Annotated[str, Query(description="AIDP API server URL")], + api_key: Annotated[str, Query(description="AIDP API key")], +) -> JSONResponse: + """Fetch ALL knowledge bases from AIDP (accumulates every page internally). + + Use this when you need the total count and want to handle pagination + entirely on the client side. + """ + try: + result = fetch_all_aidp_knowledge_bases_impl( + server_url=server_url, + api_key=api_key, + ) + return JSONResponse(status_code=HTTPStatus.OK, content=result) + except AppException: + raise + except Exception as e: + logger.exception("Failed to fetch all AIDP knowledge bases: %s", e) + raise AppException( + ErrorCode.AIDP_SERVICE_ERROR, + f"Failed to fetch all AIDP knowledge bases: {str(e)}", + ) diff --git a/backend/apps/config_app.py b/backend/apps/config_app.py index a818ec7cb..9ffadfe5e 100644 --- a/backend/apps/config_app.py +++ b/backend/apps/config_app.py @@ -33,6 +33,7 @@ from apps.monitoring_app import router as monitoring_router from apps.a2a_server_app import router as a2a_server_router from apps.haotian_app import router as haotian_router +from apps.aidp_app import router as aidp_router from apps.cas_app import router as cas_router from consts.const import IS_SPEED_MODE from services.prompt_template_service import sync_system_default_prompt_template @@ -92,3 +93,4 @@ async def sync_default_prompt_template_on_startup(): app.include_router(a2a_client_router) app.include_router(a2a_server_router) app.include_router(haotian_router) +app.include_router(aidp_router) diff --git a/backend/apps/model_managment_app.py b/backend/apps/model_managment_app.py index 53dfebb02..a92937e12 100644 --- a/backend/apps/model_managment_app.py +++ b/backend/apps/model_managment_app.py @@ -16,7 +16,10 @@ from consts.model import ( BatchCreateModelsRequest, + CapacitySuggestionFields, ModelRequest, + ModelCapacitySuggestionRequest, + ModelCapacitySuggestionResponse, ProviderModelRequest, ManageTenantModelListRequest, ManageTenantModelListResponse, @@ -28,6 +31,7 @@ ManageProviderModelListRequest, ManageProviderModelCreateRequest, ) +from consts.const import CAPACITY_SUGGESTION_ENABLED from fastapi import APIRouter, Header, Query, HTTPException from fastapi.responses import JSONResponse @@ -38,6 +42,7 @@ check_model_connectivity, verify_model_config_connectivity, ) +from services.model_capacity_suggestion_service import suggest_capacity from services.model_management_service import ( create_model_for_tenant, create_provider_models_for_tenant, @@ -49,6 +54,7 @@ list_models_for_tenant, list_llm_models_for_tenant, list_models_for_admin, + get_capacity_coverage, ) from utils.auth_utils import get_current_user_id @@ -57,6 +63,59 @@ logger = logging.getLogger("model_management_app") +def _capacity_suggestion_response_to_model(result) -> ModelCapacitySuggestionResponse: + suggestions = None + if result.suggestions is not None: + suggestions = CapacitySuggestionFields( + context_window_tokens=result.suggestions.context_window_tokens, + max_input_tokens=result.suggestions.max_input_tokens, + max_output_tokens=result.suggestions.max_output_tokens, + default_output_reserve_tokens=result.suggestions.default_output_reserve_tokens, + tokenizer_family=result.suggestions.tokenizer_family, + ) + + return ModelCapacitySuggestionResponse( + suggestions=suggestions, + match_kind=result.match_kind.value, + match_confidence=result.match_confidence.value if result.match_confidence else None, + match_explanation=result.match_explanation, + suggested_provider=result.suggested_provider, + canonical_model_name=result.canonical_model_name, + capability_profile_version=result.capability_profile_version, + capacity_source_on_accept=result.capacity_source_on_accept, + ) + + +def _suggest_capacity_for_request(request: ModelCapacitySuggestionRequest) -> ModelCapacitySuggestionResponse: + result = suggest_capacity( + model_name=request.model_name, + base_url=request.base_url, + provider_hint=request.provider_hint, + model_type=request.model_type, + api_key=request.api_key, + enabled=CAPACITY_SUGGESTION_ENABLED, + ) + return _capacity_suggestion_response_to_model(result) + + +def _capacity_suggestion_for_model_request(request: ModelRequest): + if not CAPACITY_SUGGESTION_ENABLED: + return None + + try: + suggestion_request = ModelCapacitySuggestionRequest( + model_name=request.model_name, + base_url=request.base_url, + provider_hint=request.model_factory, + api_key=request.api_key, + model_type=request.model_type, + ) + return _suggest_capacity_for_request(suggestion_request).model_dump() + except ValueError as exc: + logger.debug("Capacity suggestion unavailable for connectivity request: %s", exc) + return None + + @router.post("/create") async def create_model(request: ModelRequest, authorization: Optional[str] = Header(None)): """Create a single model record for the current tenant. @@ -90,6 +149,57 @@ async def create_model(request: ModelRequest, authorization: Optional[str] = Hea status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=str(e)) +@router.post("/suggest-capacity") +async def suggest_model_capacity( + request: ModelCapacitySuggestionRequest, + authorization: Optional[str] = Header(None), +): + """Return a non-mutating capacity suggestion for a model add/edit form. + + Response uses the shared `/model/*` envelope ({message, data}) so the + frontend service layer can unwrap it the same way as every other + `/model/*` route. Returning the bare Pydantic model broke the dialog + and coverage-banner integrations because the frontend reads + `result.data` unconditionally. + """ + try: + get_current_user_id(authorization) + result = _suggest_capacity_for_request(request) + return JSONResponse(status_code=HTTPStatus.OK, content={ + "message": "Successfully suggested model capacity", + "data": jsonable_encoder(result), + }) + except ValueError as e: + logging.error(f"Invalid capacity suggestion request: {str(e)}") + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) + except HTTPException: + raise + except Exception as e: + logging.error(f"Failed to suggest model capacity: {str(e)}") + raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=str(e)) + + +@router.get("/capacity-coverage") +async def get_model_capacity_coverage(authorization: Optional[str] = Header(None)): + """Return bare-capacity LLM/VLM coverage for the current tenant. + + Wrapped in the shared `{message, data}` envelope; see + `suggest_model_capacity` for the same rationale. + """ + try: + _, tenant_id = get_current_user_id(authorization) + result = get_capacity_coverage(tenant_id) + return JSONResponse(status_code=HTTPStatus.OK, content={ + "message": "Successfully retrieved model capacity coverage", + "data": jsonable_encoder(result), + }) + except HTTPException: + raise + except Exception as e: + logging.error(f"Failed to get model capacity coverage: {str(e)}") + raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=str(e)) + + @router.post("/provider/create") async def create_provider_model(request: ProviderModelRequest, authorization: Optional[str] = Header(None)): """Create or refresh provider models for the current tenant in memory only. @@ -338,6 +448,11 @@ async def check_temporary_model_health(request: ModelRequest): """ try: result = await verify_model_config_connectivity(request.model_dump()) + result["capacity_suggestion"] = ( + _capacity_suggestion_for_model_request(request) + if result.get("connectivity") is True + else None + ) return JSONResponse(status_code=HTTPStatus.OK, content={ "message": "Successfully verified model connectivity", "data": result diff --git a/backend/apps/skill_app.py b/backend/apps/skill_app.py index a2a3b38cf..5a67cafd5 100644 --- a/backend/apps/skill_app.py +++ b/backend/apps/skill_app.py @@ -592,6 +592,7 @@ def _build_model_config_from_tenant(tenant_id: str) -> ModelConfig: """Build ModelConfig from tenant's quick-config LLM model.""" from utils.config_utils import tenant_config_manager, get_model_name_from_config from consts.const import MODEL_CONFIG_MAPPING + from nexent.core.models.prompt_cache import resolve_prompt_cache_profile quick_config = tenant_config_manager.get_model_config( key=MODEL_CONFIG_MAPPING["llm"], @@ -600,6 +601,7 @@ def _build_model_config_from_tenant(tenant_id: str) -> ModelConfig: if not quick_config: raise ValueError("No LLM model configured for tenant") + model_factory = quick_config.get("model_factory") return ModelConfig( cite_name=quick_config.get("display_name", "default"), api_key=quick_config.get("api_key", ""), @@ -608,7 +610,8 @@ def _build_model_config_from_tenant(tenant_id: str) -> ModelConfig: temperature=0.1, top_p=0.95, ssl_verify=True, - model_factory=quick_config.get("model_factory") + model_factory=model_factory, + prompt_cache=resolve_prompt_cache_profile(model_factory), ) diff --git a/backend/consts/capability_profiles.py b/backend/consts/capability_profiles.py new file mode 100644 index 000000000..d6f30f4dd --- /dev/null +++ b/backend/consts/capability_profiles.py @@ -0,0 +1,162 @@ +"""Day-one capability profile catalog for ModelCapacityResolver. + +Source of truth: W1 ADR at +`doc/working/context-management-workstreams/W1_ADR_Capability_Catalog_Storage_and_Fingerprint.md`. + +This module owns the approved catalog data. The SDK resolver +(`sdk/nexent/core/models/capacity_resolver.py`) takes the catalog as a parameter; +it does not import this module directly. Backend services read CATALOG here and +pass it through to the resolver. + +Changes to entries: bump the per-entry `capability_profile_version` integer +suffix AND `CATALOG_REVISION` in one PR. Numerical values must be re-verified +against provider documentation at PR merge time. +""" +from __future__ import annotations + +import logging +from typing import Dict + +from nexent.core.models.capacity_resolver import CapabilityProfile, ProfileKey + +logger = logging.getLogger(__name__) + + +CATALOG_REVISION = "2026-06-23.4" + + +CATALOG: Dict[ProfileKey, CapabilityProfile] = { + ("openai", "gpt-4o"): CapabilityProfile( + provider="openai", + model_name="gpt-4o", + capability_profile_version="openai/gpt-4o@1", + window_shape="combined", + context_window_tokens=128_000, + max_output_tokens=16_384, + default_output_reserve_tokens=4_096, + tokenizer_family="o200k_base", + ), + ("openai", "gpt-4.1"): CapabilityProfile( + provider="openai", + model_name="gpt-4.1", + capability_profile_version="openai/gpt-4.1@1", + window_shape="combined", + context_window_tokens=1_000_000, + max_output_tokens=32_768, + default_output_reserve_tokens=8_192, + tokenizer_family="o200k_base", + ), + ("dashscope", "qwen-plus"): CapabilityProfile( + provider="dashscope", + model_name="qwen-plus", + capability_profile_version="dashscope/qwen-plus@1", + window_shape="combined", + context_window_tokens=131_072, + max_output_tokens=16_384, + default_output_reserve_tokens=4_096, + tokenizer_family="qwen", + ), + ("dashscope", "qwen-turbo"): CapabilityProfile( + provider="dashscope", + model_name="qwen-turbo", + capability_profile_version="dashscope/qwen-turbo@1", + window_shape="combined", + context_window_tokens=1_000_000, + max_output_tokens=16_384, + default_output_reserve_tokens=4_096, + tokenizer_family="qwen", + ), + # Sources cross-checked 2026-06-23: + # https://help.aliyun.com/zh/model-studio/models (Bailian model catalog) + # https://llm-stats.com/models/qwen3.7-max (1.0M input, 65.5K output) + ("dashscope", "qwen3.7-max"): CapabilityProfile( + provider="dashscope", + model_name="qwen3.7-max", + capability_profile_version="dashscope/qwen3.7-max@1", + window_shape="combined", + context_window_tokens=1_000_000, + max_output_tokens=65_536, + default_output_reserve_tokens=8_192, + tokenizer_family="qwen", + ), + ("dashscope", "glm-5.1"): CapabilityProfile( + provider="dashscope", + model_name="glm-5.1", + capability_profile_version="dashscope/glm-5.1@1", + window_shape="combined", + context_window_tokens=200_000, + max_output_tokens=131_072, + default_output_reserve_tokens=8_192, + tokenizer_family="chatglm", + ), + ("silicon", "Qwen/Qwen3.6-27B"): CapabilityProfile( + provider="silicon", + model_name="Qwen/Qwen3.6-27B", + capability_profile_version="silicon/qwen3.6-27b@1", + window_shape="combined", + context_window_tokens=262_144, + max_output_tokens=65_536, + default_output_reserve_tokens=8_192, + tokenizer_family="qwen", + ), + ("silicon", "Pro/moonshotai/Kimi-K2.6"): CapabilityProfile( + provider="silicon", + model_name="Pro/moonshotai/Kimi-K2.6", + capability_profile_version="silicon/kimi-k2.6@1", + window_shape="combined", + context_window_tokens=262_144, + max_output_tokens=131_072, + default_output_reserve_tokens=8_192, + tokenizer_family="moonshot", + ), + # DeepSeek official platform. Verified 2026-06-23 against + # https://api-docs.deepseek.com/zh-cn/quick_start/pricing + # (context 1M, max output 384K for both v4 models). Re-verify at PR + # merge time per the file header rule. + # + # `deepseek-chat` and `deepseek-reasoner` will be deprecated at + # 2026-07-24 23:59 (Beijing). Per DeepSeek docs they alias to + # `deepseek-v4-flash` non-thinking and thinking modes respectively, + # so their capacity profile mirrors `deepseek-v4-flash`. Remove these + # two entries after the deprecation date. + ("deepseek", "deepseek-chat"): CapabilityProfile( + provider="deepseek", + model_name="deepseek-chat", + capability_profile_version="deepseek/deepseek-chat@2", + window_shape="combined", + context_window_tokens=1_000_000, + max_output_tokens=384_000, + default_output_reserve_tokens=8_192, + tokenizer_family="deepseek", + ), + ("deepseek", "deepseek-reasoner"): CapabilityProfile( + provider="deepseek", + model_name="deepseek-reasoner", + capability_profile_version="deepseek/deepseek-reasoner@2", + window_shape="combined", + context_window_tokens=1_000_000, + max_output_tokens=384_000, + default_output_reserve_tokens=8_192, + tokenizer_family="deepseek", + ), + ("deepseek", "deepseek-v4-flash"): CapabilityProfile( + provider="deepseek", + model_name="deepseek-v4-flash", + capability_profile_version="deepseek/deepseek-v4-flash@1", + window_shape="combined", + context_window_tokens=1_000_000, + max_output_tokens=384_000, + default_output_reserve_tokens=8_192, + tokenizer_family="deepseek", + ), + ("deepseek", "deepseek-v4-pro"): CapabilityProfile( + provider="deepseek", + model_name="deepseek-v4-pro", + capability_profile_version="deepseek/deepseek-v4-pro@1", + window_shape="combined", + context_window_tokens=1_000_000, + max_output_tokens=384_000, + default_output_reserve_tokens=8_192, + tokenizer_family="deepseek", + ), +} diff --git a/backend/consts/const.py b/backend/consts/const.py index 574d550c0..11ca7f70e 100644 --- a/backend/consts/const.py +++ b/backend/consts/const.py @@ -168,6 +168,12 @@ class VectorDatabaseType(str, Enum): # Response flag when system prompts are withheld from non-ASSET_OWNER callers. AGENT_PROMPTS_HIDDEN_FLAG = "prompts_hidden" +# W11 capacity suggestion rollout flags. +CAPACITY_SUGGESTION_ENABLED = os.getenv( + "CAPACITY_SUGGESTION_ENABLED", "true").lower() in ("true", "1", "yes", "on") +CAPACITY_VISIBILITY_ENABLED = os.getenv( + "CAPACITY_VISIBILITY_ENABLED", "true").lower() in ("true", "1", "yes", "on") + # Deployment Version Configuration DEPLOYMENT_VERSION = os.getenv("DEPLOYMENT_VERSION", "speed") diff --git a/backend/consts/error_code.py b/backend/consts/error_code.py index fc94680fb..fd2987309 100644 --- a/backend/consts/error_code.py +++ b/backend/consts/error_code.py @@ -189,6 +189,12 @@ class ErrorCode(Enum): IDATA_RATE_LIMIT = "130405" # iData rate limit IDATA_RESPONSE_ERROR = "130406" # iData response error + # 05 - AIDP Service + AIDP_SERVICE_ERROR = "130501" # AIDP service error + AIDP_CONFIG_INVALID = "130502" # Invalid AIDP configuration + AIDP_CONNECTION_ERROR = "130503" # AIDP connection error + AIDP_AUTH_ERROR = "130504" # AIDP auth error + # ==================== 14 Northbound / 北向接口 ==================== # 01 - Request NORTHBOUND_REQUEST_FAILED = "140101" # Northbound request failed @@ -254,6 +260,10 @@ class ErrorCode(Enum): ErrorCode.IDATA_CONNECTION_ERROR: 502, ErrorCode.IDATA_RESPONSE_ERROR: 502, ErrorCode.IDATA_RATE_LIMIT: 429, + # AIDP (module 13) + ErrorCode.AIDP_CONFIG_INVALID: 400, + ErrorCode.AIDP_AUTH_ERROR: 401, + ErrorCode.AIDP_CONNECTION_ERROR: 502, # OAuth (module 16) ErrorCode.OAUTH_PROVIDER_NOT_CONFIGURED: 400, ErrorCode.OAUTH_PROVIDER_DISABLED: 400, diff --git a/backend/consts/error_message.py b/backend/consts/error_message.py index 59d290a52..bb3641604 100644 --- a/backend/consts/error_message.py +++ b/backend/consts/error_message.py @@ -123,6 +123,16 @@ class ErrorMessage: ErrorCode.DIFY_AUTH_ERROR: "Dify authentication failed. Please check your API key.", ErrorCode.DIFY_RATE_LIMIT: "Dify API rate limit exceeded. Please try again later.", ErrorCode.ME_CONNECTION_FAILED: "Failed to connect to ME service.", + ErrorCode.IDATA_SERVICE_ERROR: "iData service error.", + ErrorCode.IDATA_CONFIG_INVALID: "iData configuration invalid. Please check URL and API key format.", + ErrorCode.IDATA_CONNECTION_ERROR: "Failed to connect to iData. Please check network connection and URL.", + ErrorCode.IDATA_RESPONSE_ERROR: "Failed to parse iData response. Please check API URL.", + ErrorCode.IDATA_AUTH_ERROR: "iData authentication failed. Please check your API key.", + ErrorCode.IDATA_RATE_LIMIT: "iData API rate limit exceeded. Please try again later.", + ErrorCode.AIDP_SERVICE_ERROR: "AIDP service error.", + ErrorCode.AIDP_CONFIG_INVALID: "AIDP configuration invalid. Please check URL and API key format.", + ErrorCode.AIDP_CONNECTION_ERROR: "Failed to connect to AIDP. Please check network connection and URL.", + ErrorCode.AIDP_AUTH_ERROR: "AIDP authentication failed. Please check your API key.", # ==================== 14 Northbound / 北向接口 ==================== ErrorCode.NORTHBOUND_REQUEST_FAILED: "Northbound request failed.", diff --git a/backend/consts/model.py b/backend/consts/model.py index 00e5b8a0a..ac7446179 100644 --- a/backend/consts/model.py +++ b/backend/consts/model.py @@ -138,6 +138,56 @@ class ModelRequest(BaseModel): access_token: Optional[str] = None timeout_seconds: Optional[int] = None concurrency_limit: Optional[int] = None + # W1 capacity fields (see W1 ADR). All nullable; resolver applies precedence. + context_window_tokens: Optional[int] = None + max_input_tokens: Optional[int] = None + max_output_tokens: Optional[int] = None + default_output_reserve_tokens: Optional[int] = None + tokenizer_family: Optional[str] = None + capacity_source: Optional[str] = None + capability_profile_version: Optional[str] = None + + +class CapacitySuggestionFields(BaseModel): + context_window_tokens: Optional[int] = None + max_input_tokens: Optional[int] = None + max_output_tokens: Optional[int] = None + default_output_reserve_tokens: Optional[int] = None + tokenizer_family: Optional[str] = None + + +class ModelCapacitySuggestionRequest(BaseModel): + model_name: str = Field(..., min_length=1, max_length=512) + base_url: Optional[str] = None + provider_hint: Optional[str] = None + api_key: Optional[str] = None + model_type: Optional[str] = None + + +class ModelCapacitySuggestionResponse(BaseModel): + suggestions: Optional[CapacitySuggestionFields] = None + match_kind: Literal["catalog_exact", "catalog_fuzzy", "provider_discovery", "none"] + match_confidence: Optional[Literal["high", "medium", "low"]] = None + match_explanation: str + suggested_provider: Optional[str] = None + canonical_model_name: Optional[str] = None + capability_profile_version: Optional[str] = None + capacity_source_on_accept: Optional[Literal["operator"]] = None + + +class CapacityCoverageBareModel(BaseModel): + model_id: int + model_name: str + model_factory: Optional[str] = None + model_type: Literal["llm", "vlm", "vlm2", "vlm3"] + max_tokens: Optional[int] = None + suggestion_available: bool = False + + +class CapacityCoverageResponse(BaseModel): + total_llm_vlm: int + bare_count: int + bare_models: List[CapacityCoverageBareModel] = Field(default_factory=list) class ProviderModelRequest(BaseModel): @@ -256,6 +306,7 @@ class AgentRequest(BaseModel): minio_files: Optional[List[Dict[str, Any]]] = None agent_id: Optional[int] = None model_id: Optional[int] = None + requested_output_tokens: Optional[int] = Field(default=None, gt=0) version_no: Optional[int] = None is_debug: Optional[bool] = False tool_params: Optional[ToolParamsRequest] = None @@ -492,6 +543,7 @@ class AgentInfoRequest(BaseModel): model_name: Optional[str] = None model_id: Optional[int] = None max_steps: Optional[int] = Field(default=None, ge=1, le=30) + requested_output_tokens: Optional[int] = Field(default=None, gt=0) provide_run_summary: Optional[bool] = None duty_prompt: Optional[str] = None constraint_prompt: Optional[str] = None @@ -591,6 +643,7 @@ class ExportAndImportAgentInfo(BaseModel): business_description: str author: Optional[str] = None max_steps: int + requested_output_tokens: Optional[int] = Field(default=None, gt=0) provide_run_summary: bool verification_config: Optional[Dict[str, Any]] = None duty_prompt: Optional[str] = None @@ -627,6 +680,42 @@ class AgentRepositorySnapshot(ExportAndImportDataFormat): skills: Optional[List["SkillZipEntry"]] = None +class AgentRepositoryListingCreateRequest(BaseModel): + """Request body for creating a marketplace listing from an agent version.""" + icon: Optional[str] = Field(None, description="Marketplace card icon (emoji or URL)") + downloads: int = Field(0, ge=0, description="Initial download/copy count for card display") + tags: Optional[List[str]] = Field(None, description="Marketplace tags") + category_id: Optional[int] = Field(0, description="Optional marketplace category ID") + tool_count: Optional[int] = Field( + None, ge=0, description="Total tool count across all agents in the bundle" + ) + + +class AgentRepositoryCategoryItem(BaseModel): + """Marketplace category option for agent repository filtering.""" + id: int + key: str + name: str + + +class AgentRepositoryListingDetailResponse(BaseModel): + """Detailed marketplace listing payload for repository detail view.""" + agent_repository_id: int + agent_id: Optional[int] = None + name: str + display_name: Optional[str] = None + description: Optional[str] = None + author: Optional[str] = None + icon: Optional[str] = None + status: str + version_label: Optional[str] = None + downloads: int = 0 + created_at: Optional[str] = None + model_name: Optional[str] = None + duty_prompt: Optional[str] = None + tools: List[str] = Field(default_factory=list) + + class SkillZipEntry(BaseModel): """A skill bundled inside an agent export ZIP.""" skill_name: str diff --git a/backend/database/agent_db.py b/backend/database/agent_db.py index 533659b0f..9bac87381 100644 --- a/backend/database/agent_db.py +++ b/backend/database/agent_db.py @@ -237,6 +237,7 @@ def create_agent(agent_info, tenant_id: str, user_id: str): "group_ids": new_agent.group_ids, "is_new": new_agent.is_new, "enable_context_manager": new_agent.enable_context_manager, + "requested_output_tokens": new_agent.requested_output_tokens, "verification_config": new_agent.verification_config, "greeting_message": new_agent.greeting_message, "example_questions": new_agent.example_questions, @@ -273,8 +274,13 @@ def update_agent(agent_id, agent_info, user_id, version_no: int = 0): if not agent: raise ValueError("ag_tenant_agent_t Agent not found") - for key, value in filter_property(agent_info.__dict__, AgentInfo).items(): - if value is None: + agent_data = dict(agent_info.__dict__) + fields_set = getattr(agent_info, "model_fields_set", None) + if fields_set is not None and "requested_output_tokens" not in fields_set: + agent_data.pop("requested_output_tokens", None) + + for key, value in filter_property(agent_data, AgentInfo).items(): + if value is None and key != "requested_output_tokens": continue if key == "group_ids": value = convert_list_to_string(value) diff --git a/backend/database/agent_repository_db.py b/backend/database/agent_repository_db.py index a6bb4f48b..3f1b8c9dc 100644 --- a/backend/database/agent_repository_db.py +++ b/backend/database/agent_repository_db.py @@ -1,20 +1,25 @@ import logging import math -from typing import Any, Dict, List, Optional +from typing import Any, Collection, Dict, List, Optional -from sqlalchemy import func, or_, update +from sqlalchemy import and_, case, false, func, or_, true, update +from consts.const import ( + CAN_EDIT_ALL_USER_ROLES, + PERMISSION_EDIT, +) from database.client import as_dict, filter_property, get_db_session -from database.db_models import AgentRepository +from database.db_models import AgentInfo, AgentRepository, AgentVersion +from database.group_db import query_group_ids_by_user logger = logging.getLogger("agent_repository_db") -# Listing status: NOT_SHARED (未共享), PENDING_REVIEW (待审核), -# REJECTED (审核驳回), SHARED (已共享) -STATUS_NOT_SHARED = "NOT_SHARED" -STATUS_PENDING_REVIEW = "PENDING_REVIEW" -STATUS_REJECTED = "REJECTED" -STATUS_SHARED = "SHARED" +# Listing status: not_shared (未共享), pending_review (待审核), +# rejected (审核驳回), shared (已共享) +STATUS_NOT_SHARED = "not_shared" +STATUS_PENDING_REVIEW = "pending_review" +STATUS_REJECTED = "rejected" +STATUS_SHARED = "shared" VALID_REPOSITORY_STATUSES = frozenset({ STATUS_NOT_SHARED, @@ -23,6 +28,16 @@ STATUS_SHARED, }) +OWNERSHIP_ALL = "all" +OWNERSHIP_CREATED = "created" +OWNERSHIP_OTHERS = "others" + +VALID_OWNERSHIP_FILTERS = frozenset({ + OWNERSHIP_ALL, + OWNERSHIP_CREATED, + OWNERSHIP_OTHERS, +}) + _UPSERT_IMMUTABLE_FIELDS = frozenset({ "agent_id", "agent_repository_id", @@ -30,7 +45,7 @@ }) _UPSERT_SNAPSHOT_FIELDS = frozenset({ - "source_version_no", + "version_no", "name", "display_name", "description", @@ -38,7 +53,9 @@ "category_id", "tags", "tool_count", - "version_label", + "version_name", + "icon", + "downloads", "agent_info_json", }) @@ -93,13 +110,27 @@ def get_agent_repository_by_id_and_publisher( return as_dict(record) if record else None -def get_agent_repository_by_agent_id(agent_id: int) -> Optional[dict]: - """Fetch an active repository listing by root agent_id.""" +def get_agent_repository_by_agent_id( + agent_id: int, + version_no: Optional[int] = None, + *, + publisher_tenant_id: Optional[str] = None, +) -> Optional[dict]: + """Fetch an active repository listing by root agent_id and optional version.""" with get_db_session() as session: - record = session.query(AgentRepository).filter( + query = session.query(AgentRepository).filter( AgentRepository.agent_id == agent_id, AgentRepository.delete_flag != "Y", - ).first() + ) + if publisher_tenant_id is not None: + query = query.filter( + AgentRepository.publisher_tenant_id == publisher_tenant_id, + ) + if version_no is not None: + query = query.filter( + AgentRepository.version_no == version_no + ) + record = query.first() return as_dict(record) if record else None @@ -111,8 +142,8 @@ def upsert_agent_repository_record( """Insert or update a repository listing keyed by agent_id. When no record exists, inserts a new listing. When a record exists: - - Same source_version_no: updates status (and updated_by) only. - - Different source_version_no: updates all snapshot fields, preserving + - Same version_no: updates status (and updated_by) only. + - Different version_no: updates all snapshot fields, preserving agent_id, agent_repository_id, and publisher_tenant_id. Returns: @@ -122,7 +153,10 @@ def upsert_agent_repository_record( if agent_id is None: raise ValueError("agent_id is required for repository upsert") - existing = get_agent_repository_by_agent_id(int(agent_id)) + existing = get_agent_repository_by_agent_id( + int(agent_id), + publisher_tenant_id=publisher_tenant_id, + ) if not existing: repository_id = insert_agent_repository_record( repository_data=repository_data, @@ -131,8 +165,8 @@ def upsert_agent_repository_record( ) return repository_id, False - existing_version = existing.get("source_version_no") - incoming_version = repository_data.get("source_version_no") + existing_version = existing.get("version_no") + incoming_version = repository_data.get("version_no") repository_id = int(existing["agent_repository_id"]) if existing_version == incoming_version: @@ -164,99 +198,61 @@ def upsert_agent_repository_record( def list_agent_repository_summaries( + publisher_tenant_id: str, *, status: Optional[str] = None, + agent_id: Optional[int] = None, + category_id: Optional[int] = None, ) -> List[dict]: - """List all active repository summaries without heavy JSON blobs.""" + """List active repository summaries for a publisher tenant without heavy JSON blobs.""" with get_db_session() as session: query = session.query( AgentRepository.agent_repository_id, + AgentRepository.agent_id, AgentRepository.author, + AgentRepository.submitted_by, AgentRepository.name, AgentRepository.display_name, AgentRepository.description, AgentRepository.status, + AgentRepository.category_id, + AgentRepository.tags, + AgentRepository.tool_count, + AgentRepository.version_name, + AgentRepository.icon, + AgentRepository.downloads, ).filter( AgentRepository.delete_flag != "Y", + AgentRepository.publisher_tenant_id == publisher_tenant_id, ) if status: query = query.filter(AgentRepository.status == status) + if agent_id is not None: + query = query.filter(AgentRepository.agent_id == agent_id) + if category_id is not None: + query = query.filter(AgentRepository.category_id == category_id) rows = query.order_by(AgentRepository.agent_repository_id.desc()).all() return [ { "agent_repository_id": row.agent_repository_id, + "agent_id": row.agent_id, "author": row.author, + "submitted_by": row.submitted_by, "name": row.name, "display_name": row.display_name, "description": row.description, "status": row.status, + "category_id": row.category_id, + "tags": row.tags, + "tool_count": row.tool_count, + "version_name": row.version_name, + "icon": row.icon, + "downloads": row.downloads, } for row in rows ] -def query_agent_repository_list( - *, - page: int = 1, - page_size: int = 20, - search: Optional[str] = None, - tag: Optional[str] = None, - category_id: Optional[int] = None, - status: Optional[str] = STATUS_SHARED, - publisher_tenant_id: Optional[str] = None, -) -> Dict[str, Any]: - """Query repository listings with offset pagination.""" - page = max(page, 1) - page_size = max(min(page_size, 100), 1) - offset = (page - 1) * page_size - - with get_db_session() as session: - query = session.query(AgentRepository).filter( - AgentRepository.delete_flag != "Y", - ) - - if status: - query = query.filter(AgentRepository.status == status) - if publisher_tenant_id: - query = query.filter( - AgentRepository.publisher_tenant_id == publisher_tenant_id - ) - if category_id is not None: - query = query.filter(AgentRepository.category_id == category_id) - if tag: - query = query.filter(AgentRepository.tags.any(tag)) - if search: - keyword = f"%{search}%" - query = query.filter( - or_( - AgentRepository.name.ilike(keyword), - AgentRepository.display_name.ilike(keyword), - AgentRepository.description.ilike(keyword), - AgentRepository.author.ilike(keyword), - func.array_to_string(AgentRepository.tags, ",").ilike(keyword), - ) - ) - - total = query.count() - rows = ( - query.order_by(AgentRepository.agent_repository_id.desc()) - .offset(offset) - .limit(page_size) - .all() - ) - - total_pages = math.ceil(total / page_size) if total else 0 - return { - "items": [as_dict(row) for row in rows], - "pagination": { - "page": page, - "page_size": page_size, - "total": total, - "total_pages": total_pages, - }, - } - - def update_agent_repository_by_id( *, repository_id: int, @@ -269,11 +265,14 @@ def update_agent_repository_by_id( "display_name", "description", "author", + "submitted_by", "category_id", "tags", "tool_count", - "version_label", - "source_version_no", + "version_name", + "icon", + "downloads", + "version_no", "agent_info_json", "status", } @@ -305,16 +304,59 @@ def update_agent_repository_status_by_id( repository_id: int, status: str, user_id: str, + filter_publisher_tenant_id: Optional[str] = None, + publisher_tenant_id: Optional[str] = None, + publisher_user_id: Optional[str] = None, + submitted_by: Optional[str] = None, ) -> int: """Update repository listing status by primary key. Returns affected row count.""" + update_values: Dict[str, Any] = { + "status": status, + "updated_by": user_id, + } + if publisher_tenant_id is not None: + update_values["publisher_tenant_id"] = publisher_tenant_id + if publisher_user_id is not None: + update_values["publisher_user_id"] = publisher_user_id + if submitted_by is not None: + update_values["submitted_by"] = submitted_by + + with get_db_session() as session: + where_clauses = [ + AgentRepository.agent_repository_id == repository_id, + AgentRepository.delete_flag != "Y", + ] + if filter_publisher_tenant_id is not None: + where_clauses.append( + AgentRepository.publisher_tenant_id == filter_publisher_tenant_id + ) + result = session.execute( + update(AgentRepository) + .where(*where_clauses) + .values(**update_values) + ) + return int(result.rowcount or 0) + + +def reset_agent_repository_status( + *, + agent_repository_id: int, + agent_id: int, + status: str, + publisher_tenant_id: str, +) -> int: + """Set other active listings with the same agent and status to not_shared.""" with get_db_session() as session: result = session.execute( update(AgentRepository) .where( - AgentRepository.agent_repository_id == repository_id, + AgentRepository.agent_id == agent_id, + AgentRepository.status == status, + AgentRepository.agent_repository_id != agent_repository_id, + AgentRepository.publisher_tenant_id == publisher_tenant_id, AgentRepository.delete_flag != "Y", ) - .values(status=status, updated_by=user_id) + .values(status=STATUS_NOT_SHARED) ) return int(result.rowcount or 0) @@ -356,3 +398,222 @@ def list_agent_repository_by_publisher( ) rows = query.order_by(AgentRepository.agent_repository_id.desc()).all() return [as_dict(row) for row in rows] + + +def _build_group_ids_overlap_condition(user_group_ids: set[int]): + """Build SQL condition for CSV group_ids overlapping user_group_ids.""" + if not user_group_ids: + return false() + padded = func.concat(",", AgentInfo.group_ids, ",") + return or_(*(padded.like(f"%,{gid},%") for gid in user_group_ids)) + + +def _build_editable_agent_filter( + user_id: str, + *, + can_edit_all: bool, + user_group_ids: set[int], +): + """Build SQL WHERE clause for agents the user can edit.""" + if can_edit_all: + return true() + group_overlap = _build_group_ids_overlap_condition(user_group_ids) + return or_( + AgentInfo.created_by == user_id, + and_( + AgentInfo.ingroup_permission == PERMISSION_EDIT, + group_overlap, + ), + ) + + +def _resolve_editable_agent_access( + user_id: str, + user_role: str, +) -> tuple[bool, set[int], Any]: + """Resolve role-based edit access and the editable-agent SQL filter.""" + role = (user_role or "").upper() + can_edit_all = role in CAN_EDIT_ALL_USER_ROLES + user_group_ids: set[int] = set() + if not can_edit_all: + user_group_ids = set(query_group_ids_by_user(user_id) or []) + editable_filter = _build_editable_agent_filter( + user_id, + can_edit_all=can_edit_all, + user_group_ids=user_group_ids, + ) + return can_edit_all, user_group_ids, editable_filter + + +def _build_ownership_filter(user_id: str, ownership_filter: str): + """Build SQL WHERE clause for mine-tab ownership filtering.""" + if ownership_filter == OWNERSHIP_CREATED: + return AgentInfo.created_by == user_id + if ownership_filter == OWNERSHIP_OTHERS: + return or_( + AgentInfo.created_by != user_id, + AgentInfo.created_by.is_(None), + ) + return true() + + +def _build_editable_agent_base_filters( + tenant_id: str, + editable_filter: Any, +) -> tuple[Any, ...]: + """Shared base filters for editable draft agents in a tenant.""" + return ( + AgentInfo.tenant_id == tenant_id, + AgentInfo.version_no == 0, + AgentInfo.delete_flag != "Y", + AgentInfo.enabled.is_(True), + editable_filter, + ) + + +def list_agent_repository_by_agent_ids( + agent_ids: List[int], + *, + statuses: Collection[str], + publisher_tenant_id: str, +) -> List[dict]: + """List repository rows for the given agents, scoped to publisher tenant and statuses.""" + if not agent_ids: + return [] + + status_list = list(statuses) + with get_db_session() as session: + rows = ( + session.query( + AgentRepository.agent_repository_id, + AgentRepository.agent_id, + AgentRepository.status, + AgentRepository.version_no, + AgentRepository.version_name, + AgentRepository.create_time, + ) + .filter( + AgentRepository.delete_flag != "Y", + AgentRepository.publisher_tenant_id == publisher_tenant_id, + AgentRepository.agent_id.in_(agent_ids), + AgentRepository.status.in_(status_list), + ) + .order_by( + AgentRepository.agent_id, + AgentRepository.create_time.desc(), + ) + .all() + ) + + return [ + { + "agent_repository_id": row.agent_repository_id, + "agent_id": row.agent_id, + "status": row.status, + "version_no": row.version_no, + "version_name": row.version_name, + "create_time": row.create_time, + } + for row in rows + ] + + +def list_editable_agents_for_user( + tenant_id: str, + user_id: str, + *, + user_role: str, + ownership_filter: str = OWNERSHIP_ALL, +) -> List[dict]: + """List draft agents in a tenant that the user can edit. + + Queries version_no=0 rows and returns agent_id, name, display_name, description, + current_version_no, and the current published version_name and create_time + (via LEFT JOIN on ag_tenant_agent_version_t) for agents where permission resolves to EDIT. + """ + _, _, editable_filter = _resolve_editable_agent_access(user_id, user_role) + ownership_clause = _build_ownership_filter(user_id, ownership_filter) + + with get_db_session() as session: + rows = ( + session.query( + AgentInfo.agent_id, + AgentInfo.name, + AgentInfo.display_name, + AgentInfo.description, + AgentInfo.current_version_no, + AgentInfo.created_by, + AgentVersion.version_name, + AgentVersion.create_time, + ) + .outerjoin( + AgentVersion, + and_( + AgentInfo.agent_id == AgentVersion.agent_id, + AgentInfo.current_version_no == AgentVersion.version_no, + AgentInfo.tenant_id == AgentVersion.tenant_id, + AgentVersion.delete_flag == "N", + ), + ) + .filter( + *_build_editable_agent_base_filters(tenant_id, editable_filter), + ownership_clause, + ) + .order_by(AgentInfo.create_time.desc()) + .all() + ) + + return [ + { + "agent_id": row.agent_id, + "name": row.name, + "display_name": row.display_name, + "description": row.description, + "current_version_no": row.current_version_no, + "created_by": row.created_by, + "version_name": row.version_name, + "version_create_time": row.create_time, + } + for row in rows + ] + + +def count_editable_agents_by_ownership( + tenant_id: str, + user_id: str, + *, + user_role: str, +) -> Dict[str, int]: + """Count editable draft agents grouped by ownership for mine-tab badges.""" + _, _, editable_filter = _resolve_editable_agent_access(user_id, user_role) + created_case = case( + (AgentInfo.created_by == user_id, 1), + else_=0, + ) + others_case = case( + ( + or_( + AgentInfo.created_by != user_id, + AgentInfo.created_by.is_(None), + ), + 1, + ), + else_=0, + ) + + with get_db_session() as session: + row = ( + session.query( + func.count(AgentInfo.agent_id), + func.coalesce(func.sum(created_case), 0), + func.coalesce(func.sum(others_case), 0), + ) + .filter(*_build_editable_agent_base_filters(tenant_id, editable_filter)) + .one() + ) + + return { + "all": int(row[0] or 0), + "created": int(row[1] or 0), + "others": int(row[2] or 0), + } diff --git a/backend/database/conversation_db.py b/backend/database/conversation_db.py index 2d06bb9be..e401beda9 100644 --- a/backend/database/conversation_db.py +++ b/backend/database/conversation_db.py @@ -623,9 +623,18 @@ def get_conversation_history(conversation_id: int, user_id: Optional[str] = None } +def _image_exists(session, message_id: int, image_url: str) -> bool: + stmt = select(ConversationSourceImage).where( + ConversationSourceImage.message_id == message_id, + ConversationSourceImage.image_url == image_url, + ConversationSourceImage.delete_flag == 'N' + ).limit(1) + return session.execute(stmt).scalar_one_or_none() is not None + + def create_source_image(image_data: Dict[str, Any], user_id: Optional[str] = None) -> int: """ - Create image source reference + Create image source reference (skips if the same message_id + image_url already exists). Args: image_data: Dictionary containing image data, must include the following fields: @@ -634,17 +643,22 @@ def create_source_image(image_data: Dict[str, Any], user_id: Optional[str] = Non user_id: Reserved parameter for created_by and updated_by fields Returns: - int: Newly created image ID (auto-increment ID) + int: Newly created image ID (auto-increment ID), or -1 if skipped due to duplicate """ with get_db_session() as session: # Ensure message_id is of integer type message_id = int(image_data['message_id']) + image_url = image_data['image_url'] + + # Skip duplicate: same message_id + image_url already in DB + if _image_exists(session, message_id, image_url): + return -1 # Prepare data dictionary data = { "message_id": message_id, "conversation_id": image_data.get('conversation_id'), - "image_url": image_data['image_url'], + "image_url": image_url, "delete_flag": 'N', # Use the database's CURRENT_TIMESTAMP function "create_time": func.current_timestamp() diff --git a/backend/database/db_models.py b/backend/database/db_models.py index 5450b5f74..eed1b3a62 100644 --- a/backend/database/db_models.py +++ b/backend/database/db_models.py @@ -188,6 +188,20 @@ class ModelRecord(TableBase): Integer, doc="Request timeout in seconds for this model. Default is 120 seconds.") concurrency_limit = Column( Integer, doc="Maximum concurrent requests for this model. Default is null (unlimited).") + context_window_tokens = Column( + Integer, doc="Total combined input/output context window in tokens, when the provider uses a combined window. Nullable.") + max_input_tokens = Column( + Integer, doc="Provider hard input-token limit when distinct from the combined window. Nullable.") + max_output_tokens = Column( + Integer, doc="Provider-supported or operator-configured completion-output cap. Replaces the ambiguous LLM meaning of max_tokens. Nullable.") + default_output_reserve_tokens = Column( + Integer, doc="Default output allowance reserved per request before constructing input context. Nullable.") + tokenizer_family = Column( + String(100), doc="Token-counting strategy or provider/model tokenizer identifier mapped via tokenizer_registry. Nullable.") + capacity_source = Column( + String(100), doc="Source of the persisted capacity value. Optional values: operator, profile, provider_candidate, legacy, unknown.") + capability_profile_version = Column( + String(100), doc="Version of the approved provider/model capability profile used by the request, e.g. openai/gpt-4o@1.") class ModelMonitoringRecord(SimpleTableBase): @@ -237,6 +251,69 @@ class ModelMonitoringRecord(SimpleTableBase): input_tokens = Column(Integer, doc="Number of input tokens") output_tokens = Column(Integer, doc="Number of output tokens") total_tokens = Column(Integer, doc="Total tokens (input + output)") + context_window_tokens = Column( + Integer, doc="Resolved total combined model context window for this request" + ) + default_output_reserve_tokens = Column( + Integer, doc="Default output allowance reserved before input context construction" + ) + capability_profile_version = Column( + String(100), doc="Version of the resolved capacity profile for this request" + ) + capacity_source = Column( + String(100), doc="Dominant source of resolved capacity fields for this request" + ) + requested_output_tokens = Column( + Integer, doc="Output tokens requested or reserved during capacity resolution" + ) + provider_input_limit_tokens = Column( + Integer, doc="Resolved provider input-token limit used by context management" + ) + tokenizer_family = Column( + String(100), doc="Tokenizer family used for request token counting" + ) + counting_mode = Column( + String(20), doc="Token counting mode for the request: exact or estimated" + ) + unknown_capabilities = Column( + JSONB, doc="Structured list of capacity capabilities unknown at resolution time" + ) + capacity_fingerprint = Column( + String(64), doc="Fingerprint of the resolved model capacity snapshot" + ) + budget_fingerprint = Column( + String(64), doc="Fingerprint of the resolved W2 safe input budget snapshot" + ) + budget_w1_fingerprint = Column( + String(64), doc="W1 capacity fingerprint consumed by the W2 budget snapshot" + ) + budget_requested_output_tokens = Column( + Integer, doc="W2 trusted requested output tokens used at dispatch" + ) + budget_output_reserve_source = Column( + String(32), doc="Source of the W2 requested output token reserve" + ) + budget_provider_input_limit_tokens = Column( + Integer, doc="Provider input limit after applying the W2 output reserve" + ) + budget_uncertainty_reserve_tokens = Column( + Integer, doc="Additional W2 uncertainty reserve deducted from input budget" + ) + budget_uncertainty_reserve_basis = Column( + String(64), doc="Basis used for the W2 uncertainty reserve" + ) + budget_soft_limit_ratio = Column( + Float, doc="W2 soft input budget ratio" + ) + budget_soft_input_budget_tokens = Column( + Integer, doc="W2 soft input budget where proactive compression begins" + ) + budget_hard_input_budget_tokens = Column( + Integer, doc="W2 hard input budget consumed by W3 final fit" + ) + budget_warnings = Column( + JSONB, doc="Structured W2 budget warnings active for this request" + ) generation_rate = Column( Float, doc="Token generation rate (tokens per second)") is_streaming = Column( @@ -332,7 +409,14 @@ class AgentInfo(TableBase): is_new = Column(Boolean, default=False, doc="Whether this agent is marked as new for the user") current_version_no = Column(Integer, nullable=True, doc="Current published version number. NULL means no version published yet") ingroup_permission = Column(String(30), doc="In-group permission: EDIT, READ_ONLY, PRIVATE") - enable_context_manager = Column(Boolean, default=False, doc="Whether to enable context management (compression) for this agent") + requested_output_tokens = Column( + Integer, + doc=( + "Per-agent override for W2 requested_output_tokens. NULL means " + "inherit the resolved model-level default." + ), + ) + enable_context_manager = Column(Boolean, default=True, doc="Whether to enable context management (compression) for this agent") verification_config = Column(JSONB, doc="Layered ReAct self-verification configuration") greeting_message = Column(Text, doc="Agent greeting message displayed on chat initial screen") example_questions = Column(JSONB, doc="List of example questions for starting a conversation with this agent") @@ -718,23 +802,27 @@ class AgentRepository(TableBase): publisher_user_id = Column(String(100), nullable=False, doc="Publisher user ID") agent_id = Column(Integer, nullable=False, doc="Root agent ID from ag_tenant_agent_t; upsert key") - source_version_no = Column(Integer, nullable=False, - doc="Published version number frozen at share time") + version_no = Column(Integer, nullable=False, + doc="Published version number frozen at share time") name = Column(String(100), nullable=False, doc="Root agent programmatic name for display and search") display_name = Column(String(100), doc="Root agent display name") description = Column(Text, doc="Root agent description") author = Column(String(100), doc="Agent author") + submitted_by = Column(String(100), doc="Submitter email when listing enters pending_review") category_id = Column(Integer, doc="Optional marketplace category ID") tags = Column(ARRAY(Text), doc="Marketplace tags") tool_count = Column(Integer, doc="Total tool count across all agents in the bundle (display only)") - version_label = Column(String(100), - doc="Repository entry version label for display (e.g. v1.0)") + icon = Column(String(100), doc="Marketplace card icon (emoji or URL)") + downloads = Column(Integer, default=0, + doc="Marketplace download/copy count for card display") + version_name = Column(String(100), + doc="Repository entry version name for display (from ag_tenant_agent_version_t)") agent_info_json = Column(JSONB, nullable=False, doc="Frozen ExportAndImportDataFormat snapshot with optional skills") - status = Column(String(30), default="NOT_SHARED", - doc="Listing status: NOT_SHARED (未共享) / PENDING_REVIEW (待审核) / REJECTED (审核驳回) / SHARED (已共享)") + status = Column(String(30), default="not_shared", + doc="Listing status: not_shared (未共享) / pending_review (待审核) / rejected (审核驳回) / shared (已共享)") class UserTokenInfo(TableBase): diff --git a/backend/database/tool_db.py b/backend/database/tool_db.py index 4d34ede9b..907dfd012 100644 --- a/backend/database/tool_db.py +++ b/backend/database/tool_db.py @@ -47,6 +47,13 @@ def create_or_update_tool_by_tool_info(tool_info, tenant_id: str, user_id: str, tool_info_dict = tool_info.__dict__ | { "tenant_id": tenant_id, "user_id": user_id, "version_no": version_no} + # Filter out null values from params to avoid saving nulls to database + if 'params' in tool_info_dict and tool_info_dict['params'] is not None: + tool_info_dict['params'] = { + k: v for k, v in tool_info_dict['params'].items() + if v is not None + } + with get_db_session() as session: # Query if there is an existing ToolInstance # Note: Do not filter by user_id to avoid creating duplicate instances @@ -71,7 +78,7 @@ def create_or_update_tool_by_tool_info(tool_info, tenant_id: str, user_id: str, session.add(new_tool_instance) session.flush() # Flush to get the ID tool_instance = new_tool_instance - return tool_instance + return as_dict(tool_instance) def query_all_tools(tenant_id: str): @@ -258,7 +265,11 @@ def add_tool_field(tool_info): tool_params = tool.params for ele in tool_params: param_name = ele["name"] - ele["default"] = tool_info["params"].get(param_name) + instance_value = tool_info["params"].get(param_name) + # Only set default if instance value is not None + # This prevents null values from being saved to database and returned as defaults + if instance_value is not None: + ele["default"] = instance_value tool_dict = as_dict(tool) tool_dict["params"] = tool_params diff --git a/backend/prompts/managed_system_prompt_template_en.yaml b/backend/prompts/managed_system_prompt_template_en.yaml index 62e16e946..b42379d23 100644 --- a/backend/prompts/managed_system_prompt_template_en.yaml +++ b/backend/prompts/managed_system_prompt_template_en.yaml @@ -48,6 +48,65 @@ system_prompt: |- Security Protection: Do not respond to requests involving weapon manufacturing, cyberattacks, fraud, malware, or other dangerous activities; Ethical Guidelines: Refuse hate speech, discriminatory content, and any requests that violate social morals and commonly accepted ethical standards. + {%- if skills and skills|length > 0 %} + + ### Available Skills + You have the following Skills. Skills are predefined professional capability modules with detailed execution guides and optional additional scripts. + + {%- for skill in skills %} + + {{ skill.name }} + {{ skill.description }} + + {%- endfor %} + + + **Skill Usage Process**: + 1. After receiving a user request, first examine the description of each skill in `` to determine if there is a matching skill. + 2. **Load Skill**: Choose the appropriate reading method based on the scenario: + - **First-time load**: Call `read_skill_md("skill_name")` to read the complete execution guide (defaults to reading SKILL.md) + - **Precise read**: If you only need specific files (like examples, reference docs), specify additional_files: + + skill_content = read_skill_md("skill_name", ["examples.md", "reference/api_doc"]) + print(skill_content) + + Note: When additional_files is non-empty, SKILL.md is no longer auto-read. If you need both, explicitly specify it. + - **Load skill config**: If the skill needs configuration variables, call `read_skill_config("skill_name")` to read the config string, convert to dict via `json.loads`, then access values: + + import json + config = json.loads(read_skill_config("skill_name")) + # Example: {"key_a": {"key2": "value2"}, "others": {...}} + value = config["key1"]["key2"] + print(value) + + 3. **Follow Skill Guide**: After skill content is injected, strictly follow its steps. Do not skip steps or replace with your own code. + 4. **Execute Skill Script**: If the skill guide references additional scripts (like ``), call: + + result = run_skill_script("skill_name", "script_path") + print(result) + + For scripts needing extra params, pass them as a command-line string per the script's calling instructions. + Example for --param1 value1 --flag: + + result = run_skill_script("skill_name", "script_path", "--param1 value1 --flag") + print(result) + + Note: Only execute script paths explicitly declared in the skill guide. Never construct paths yourself. + + 5. **Integrate Output**: Generate the final answer based on the skill guide's output format and script execution results. + + 6. **Handle References**: When the skill content has reference markers or needs to reference other files, identify and call read_skill_md again: + - **Reference template recognition**: Look for patterns like `` or natural-language references ("see examples.md", "refer to reference/api_doc") + - **Auto-complete**: After discovering a reference, try reading the referenced file for more info + - **Example**: + + # Skill content says "see examples.md for detailed examples" + additional_info = read_skill_md("skill_name", ["examples.md"]) + print(additional_info) + + + {%- endif %} + ### Execution Process To solve tasks, you must plan forward through a series of steps in a loop of 'Think:' and 'Code:' sequences. **IMPORTANT: You must NOT output 'Observe Results:' before code execution. Observation results can ONLY be generated after code execution.** @@ -129,6 +188,22 @@ system_prompt: |- - No tools are currently available {%- endif %} + {%- if skills and skills|length > 0 %} + - You have the skills listed in `` above. Scripts referenced in skills are called via the `run_skill_script()` function, which is provided by the platform and does not need to be imported. + + ### Skill Usage Requirements + 1. **Skill First**: If a user request matches a skill's description, you must first call `read_skill_md()` to load the skill guide, then follow it. Do not skip the skill and write your own code to solve it. + 2. **Faithful Execution**: After reading the skill content, strictly follow the steps in the skill guide. Do not modify the process, skip steps, or replace the skill-defined workflow with generic code. + 3. **Script Calling Standards**: Only use the `run_skill_script` tool to execute scripts explicitly required by the skill guide. The `skill_name` and `script_path` passed in must exactly match the declarations in the skill guide. Do not construct or guess paths yourself. For scripts requiring additional parameters, pass the parameters as a command-line string to `run_skill_script`. + 4. **Failure Fallback**: If `read_skill_md` returns an error or `run_skill_script` fails, explain the situation to the user and try to provide an alternative using general reasoning. + 5. **Skill Composition**: If a task requires multiple skills working together, load and execute them in logical dependency order. The output of one skill can serve as the input for the next. + + + {%- else %} + - No skills are currently available + {%- endif %} + + ### Resource Usage Requirements {{ constraint }} diff --git a/backend/prompts/manager_system_prompt_template_en.yaml b/backend/prompts/manager_system_prompt_template_en.yaml index d44ed9a71..c4c18d16d 100644 --- a/backend/prompts/manager_system_prompt_template_en.yaml +++ b/backend/prompts/manager_system_prompt_template_en.yaml @@ -48,6 +48,68 @@ system_prompt: |- Security Protection: Do not respond to requests involving weapon manufacturing, cyberattacks, fraud, malware, or other dangerous activities; Ethical Guidelines: Refuse hate speech, discriminatory content, and any requests that violate social morals and commonly accepted ethical standards. + {%- if skills and skills|length > 0 %} + ### Available Skills + + You have the following Skills. Skills are predefined professional capability modules with detailed execution guides and optional additional scripts. + + + {%- for skill in skills %} + + {{ skill.name }} + {{ skill.description }} + + {%- endfor %} + + + **Skill Usage Process**: + 1. After receiving a user request, first examine the description of each skill in `` to determine if there is a matching skill. + 2. **Load Skill**: Choose the appropriate reading method based on the scenario: + - **First-time load**: Call `read_skill_md("skill_name")` to read the complete execution guide (defaults to reading SKILL.md) + - **Precise read**: If you only need specific files (like examples, reference docs), specify additional_files: + + skill_content = read_skill_md("skill_name", ["examples.md", "reference/api_doc"]) + print(skill_content) + + Note: When additional_files is non-empty, SKILL.md is no longer auto-read. If you need both, explicitly specify it. + + - **Load skill config**: If the skill needs configuration variables, call `read_skill_config("skill_name")` to read the config string, convert to dict via `json.loads`, then access values: + + import json + config = json.loads(read_skill_config("skill_name")) + # Example: {"key_a": {"key2": "value2"}, "others": {...}} + value = config["key1"]["key2"] + print(value) + + + 3. **Follow Skill Guide**: After skill content is injected, strictly follow its steps. Do not skip steps or replace with your own code. + + 4. **Execute Skill Script**: If the skill guide references additional scripts (like ``), call: + + result = run_skill_script("skill_name", "script_path") + print(result) + + For scripts needing extra params, pass them as a command-line string per the script's calling instructions. + Example for --param1 value1 --flag: + + result = run_skill_script("skill_name", "script_path", "--param1 value1 --flag") + print(result) + + Note: Only execute script paths explicitly declared in the skill guide. Never construct paths yourself. + + 5. **Integrate Output**: Generate the final answer based on the skill guide's output format and script execution results. + + 6. **Handle References**: When the skill content has reference markers or needs to reference other files, identify and call read_skill_md again: + - **Reference template recognition**: Look for patterns like `` or natural-language references ("see examples.md", "refer to reference/api_doc") + - **Auto-complete**: After discovering a reference, try reading the referenced file for more info + - **Example**: + + # Skill content says "see examples.md for detailed examples" + additional_info = read_skill_md("skill_name", ["examples.md"]) + print(additional_info) + + {%- endif %} + ### Execution Process To solve tasks, you must plan forward through a series of steps in a loop of 'Think:' and 'Code:' sequences. **IMPORTANT: You must NOT output 'Observe Results:' before code execution. Observation results can ONLY be generated after code execution.** @@ -169,7 +231,21 @@ system_prompt: |- - No agents are currently available {%- endif %} - ### Resource Usage Requirements + 3. Skills + {%- if skills and skills|length > 0 %} + - You have the skills listed in `` above. Scripts referenced in skills are called via the `run_skill_script()` function, which is provided by the platform and does not need to be imported. + + ### Skill Usage Requirements + 1. **Skill First**: If a user request matches a skill's description, you must first call `read_skill_md()` to load the skill guide, then follow it. Do not skip the skill and write your own code to solve it. + 2. **Faithful Execution**: After reading the skill content, strictly follow the steps in the skill guide. Do not modify the process, skip steps, or replace the skill-defined workflow with generic code. + 3. **Script Calling Standards**: Only use the `run_skill_script` tool to execute scripts explicitly required by the skill guide. The `skill_name` and `script_path` passed in must exactly match the declarations in the skill guide. Do not construct or guess paths yourself. For scripts requiring additional parameters, pass the parameters as a command-line string to `run_skill_script`. + 4. **Failure Fallback**: If `read_skill_md` returns an error or `run_skill_script` fails, explain the situation to the user and try to provide an alternative using general reasoning. + 5. **Skill Composition**: If a task requires multiple skills working together, load and execute them in logical dependency order. The output of one skill can serve as the input for the next. + {%- else %} + - No skills are currently available + {%- endif %} + + ### Resource Usage Requirements {{ constraint }} ### Python Code Specifications diff --git a/backend/services/agent_repository_service.py b/backend/services/agent_repository_service.py index 87649bcd1..1c1b29426 100644 --- a/backend/services/agent_repository_service.py +++ b/backend/services/agent_repository_service.py @@ -1,20 +1,30 @@ import logging -from typing import Any, Dict, Optional +from typing import Any, Dict, FrozenSet, List, Optional, Tuple -from consts.const import ASSET_OWNER_TENANT_ID +from consts.exceptions import UnauthorizedError from consts.model import AgentRepositorySnapshot from database.agent_db import search_agent_info_by_agent_id from database.agent_version_db import search_version_by_version_no from database.agent_repository_db import ( + STATUS_NOT_SHARED, STATUS_PENDING_REVIEW, + STATUS_REJECTED, + STATUS_SHARED, + OWNERSHIP_ALL, + VALID_OWNERSHIP_FILTERS, VALID_REPOSITORY_STATUSES, + count_editable_agents_by_ownership, get_agent_repository_by_agent_id, - get_agent_repository_by_id, + get_agent_repository_by_id_and_publisher, insert_agent_repository_record, + list_agent_repository_by_agent_ids, list_agent_repository_summaries, + list_editable_agents_for_user, + reset_agent_repository_status, update_agent_repository_by_id, update_agent_repository_status_by_id, ) +from database.user_tenant_db import get_user_tenant_by_user_id from services.agent_service import ( collect_skill_zip_entries, export_agent_dict_for_repository_impl, @@ -24,15 +34,53 @@ logger = logging.getLogger("agent_repository_service") +_SU_STATUS_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_PENDING_REVIEW, STATUS_REJECTED), + (STATUS_PENDING_REVIEW, STATUS_SHARED), + (STATUS_SHARED, STATUS_NOT_SHARED), +}) + +_PUBLISHER_STATUS_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_NOT_SHARED, STATUS_PENDING_REVIEW), + (STATUS_REJECTED, STATUS_PENDING_REVIEW), + (STATUS_PENDING_REVIEW, STATUS_NOT_SHARED), + (STATUS_REJECTED, STATUS_NOT_SHARED), + (STATUS_SHARED, STATUS_NOT_SHARED), +}) + +_PUBLISHER_RESUBMIT_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_NOT_SHARED, STATUS_PENDING_REVIEW), + (STATUS_REJECTED, STATUS_PENDING_REVIEW), +}) + +_ADMIN_REVIEW_STATUS_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_PENDING_REVIEW, STATUS_REJECTED), + (STATUS_PENDING_REVIEW, STATUS_SHARED), +}) + +_REPOSITORY_STATUS_PRIORITY: Dict[str, int] = { + STATUS_SHARED: 4, + STATUS_PENDING_REVIEW: 3, + STATUS_REJECTED: 2, + STATUS_NOT_SHARED: 1, +} + +_MAX_LISTING_TAGS = 5 +_MAX_LISTING_TAG_LENGTH = 20 +_MAX_LISTING_ICON_LENGTH = 32 + _UPDATE_SNAPSHOT_FIELDS = ( "display_name", "description", "author", + "submitted_by", "category_id", "tags", "tool_count", - "version_label", - "source_version_no", + "version_name", + "icon", + "downloads", + "version_no", "agent_info_json", "status", ) @@ -42,33 +90,398 @@ def _to_summary_item(record: Dict[str, Any]) -> Dict[str, Any]: """Map a DB record to a lightweight marketplace summary item.""" return { "agent_repository_id": record.get("agent_repository_id"), + "agent_id": record.get("agent_id"), "author": record.get("author"), + "submitted_by": record.get("submitted_by"), "name": record.get("name"), "display_name": record.get("display_name"), "description": record.get("description"), "status": record.get("status"), + "category_id": record.get("category_id"), + "tags": record.get("tags") or [], + "tool_count": record.get("tool_count"), + "version_label": record.get("version_name"), + "icon": record.get("icon"), + "downloads": record.get("downloads") or 0, } +def _deduplicate_repository_summaries_by_agent_id( + records: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Keep one repository summary per agent using marketplace status priority.""" + selected_records: Dict[Tuple[str, Any], Dict[str, Any]] = {} + + for record in records: + agent_id = record.get("agent_id") + dedupe_key = ( + ("agent", agent_id) + if agent_id is not None + else ("repository", record.get("agent_repository_id")) + ) + current = selected_records.get(dedupe_key) + if current is None or _repository_summary_rank(record) > _repository_summary_rank(current): + selected_records[dedupe_key] = record + + return sorted( + selected_records.values(), + key=lambda record: int(record.get("agent_repository_id") or 0), + reverse=True, + ) + + +def _repository_summary_rank(record: Dict[str, Any]) -> Tuple[int, int]: + """Rank summaries by status priority, then newest repository ID.""" + return ( + _REPOSITORY_STATUS_PRIORITY.get(str(record.get("status") or ""), 0), + int(record.get("agent_repository_id") or 0), + ) + + def list_agent_repository_listings_impl( + tenant_id: str, *, status: Optional[str] = None, + agent_id: Optional[int] = None, + deduplicate_by_agent_id: bool = True, + category_id: Optional[int] = None, ) -> Dict[str, Any]: - """List all repository listings with optional status filter.""" + """List repository listings for the caller tenant with optional status filter.""" if status is not None and status not in VALID_REPOSITORY_STATUSES: raise ValueError( f"Invalid status '{status}'; must be one of: " f"{', '.join(sorted(VALID_REPOSITORY_STATUSES))}" ) - records = list_agent_repository_summaries(status=status) + records = list_agent_repository_summaries( + publisher_tenant_id=tenant_id, + status=status, + agent_id=agent_id, + category_id=category_id, + ) + if deduplicate_by_agent_id: + records = _deduplicate_repository_summaries_by_agent_id(records) return {"items": [_to_summary_item(record) for record in records]} +def _normalize_listing_tags(tags: Any) -> List[str]: + """Trim, deduplicate, and validate marketplace listing tags.""" + if not isinstance(tags, list): + raise ValueError("tags must be a list of strings") + + normalized: List[str] = [] + seen: set[str] = set() + for raw_tag in tags: + if not isinstance(raw_tag, str): + raise ValueError("tags must be a list of strings") + tag = raw_tag.strip() + if not tag: + continue + if len(tag) > _MAX_LISTING_TAG_LENGTH: + raise ValueError( + f"Each tag must be at most {_MAX_LISTING_TAG_LENGTH} characters" + ) + if tag in seen: + continue + seen.add(tag) + normalized.append(tag) + + if not normalized: + raise ValueError("tags must contain at least one non-empty tag") + if len(normalized) > _MAX_LISTING_TAGS: + raise ValueError(f"tags must contain at most {_MAX_LISTING_TAGS} items") + return normalized + + +def _validate_card_fields(repository_data: Dict[str, Any]) -> None: + """Validate marketplace card fields required for listing submission.""" + icon = repository_data.get("icon") + if not icon or not isinstance(icon, str) or not icon.strip(): + raise ValueError("icon is required and must be a non-empty string") + if len(icon.strip()) > _MAX_LISTING_ICON_LENGTH: + raise ValueError( + f"icon must be at most {_MAX_LISTING_ICON_LENGTH} characters" + ) + + category_id = repository_data.get("category_id") + if category_id is None or not isinstance(category_id, int): + raise ValueError("category_id is required and must be an integer") + + tags = repository_data.get("tags") + if tags is None: + raise ValueError("tags is required for marketplace listing submission") + repository_data["tags"] = _normalize_listing_tags(tags) + + +_MY_AGENT_REPOSITORY_STATUSES = frozenset({ + STATUS_SHARED, + STATUS_PENDING_REVIEW, + STATUS_REJECTED, +}) + + +def _reset_repository_peer_statuses( + *, + agent_repository_id: int, + agent_id: int, + status: str, + publisher_tenant_id: str, +) -> None: + """Reset peer listings with the same status; also clear rejected when submitting.""" + reset_agent_repository_status( + agent_repository_id=agent_repository_id, + agent_id=agent_id, + status=status, + publisher_tenant_id=publisher_tenant_id, + ) + if status == STATUS_PENDING_REVIEW: + reset_agent_repository_status( + agent_repository_id=agent_repository_id, + agent_id=agent_id, + status=STATUS_REJECTED, + publisher_tenant_id=publisher_tenant_id, + ) + + +def _to_repository_info_item(record: Dict[str, Any]) -> Dict[str, Any]: + """Map a repository DB row to a my-agents repository_info entry.""" + return { + "agent_repository_id": record.get("agent_repository_id"), + "status": record.get("status"), + "version_no": record.get("version_no"), + "version_label": record.get("version_name"), + "create_time": _serialize_created_at(record.get("create_time")), + } + + +def list_my_editable_agents_impl( + tenant_id: str, + user_id: str, + ownership: str = OWNERSHIP_ALL, +) -> Dict[str, Any]: + """List editable draft agents for the current user with repository listing info.""" + normalized_ownership = (ownership or OWNERSHIP_ALL).strip().lower() + if normalized_ownership not in VALID_OWNERSHIP_FILTERS: + raise ValueError( + f"Invalid ownership filter: {ownership}. " + f"Allowed values: {', '.join(sorted(VALID_OWNERSHIP_FILTERS))}." + ) + + user_tenant_record = get_user_tenant_by_user_id(user_id) or {} + user_role = str(user_tenant_record.get("user_role") or "").upper() + + counts = count_editable_agents_by_ownership( + tenant_id, + user_id, + user_role=user_role, + ) + agents = list_editable_agents_for_user( + tenant_id, + user_id, + user_role=user_role, + ownership_filter=normalized_ownership, + ) + agent_ids = [int(agent["agent_id"]) for agent in agents if agent.get("agent_id") is not None] + + repository_by_agent_id: Dict[int, List[Dict[str, Any]]] = {} + if agent_ids: + repository_records = list_agent_repository_by_agent_ids( + agent_ids, + statuses=_MY_AGENT_REPOSITORY_STATUSES, + publisher_tenant_id=tenant_id, + ) + for record in repository_records: + agent_id = record.get("agent_id") + if agent_id is None: + continue + repository_by_agent_id.setdefault(int(agent_id), []).append( + _to_repository_info_item(record) + ) + + items = [ + { + "agent_id": agent.get("agent_id"), + "name": agent.get("display_name") or agent.get("name"), + "description": agent.get("description"), + "current_version_no": agent.get("current_version_no"), + "version_label": agent.get("version_name"), + "version_create_time": _serialize_created_at(agent.get("version_create_time")), + "repository_info": repository_by_agent_id.get(int(agent["agent_id"]), []) + if agent.get("agent_id") is not None + else [], + } + for agent in agents + ] + + return { + "items": items, + "counts": counts, + } + + +def _resolve_submitter_email(user_id: str) -> Optional[str]: + """Resolve submitter email from user_tenant_t for pending_review listings.""" + user_tenant = get_user_tenant_by_user_id(user_id) or {} + email = str(user_tenant.get("user_email") or "").strip() + return email or None + + +def _extract_root_agent_from_snapshot(agent_info_json: Any) -> Dict[str, Any]: + """Resolve the root agent entry from a frozen repository snapshot.""" + if not isinstance(agent_info_json, dict): + return {} + root_agent_id = agent_info_json.get("agent_id") + agent_info_map = agent_info_json.get("agent_info") + if root_agent_id is None or not isinstance(agent_info_map, dict): + return {} + return ( + agent_info_map.get(str(root_agent_id)) + or agent_info_map.get(root_agent_id) + or {} + ) + + +def _extract_tool_names(root_agent: Dict[str, Any]) -> List[str]: + """Collect display tool names from a root agent snapshot entry.""" + tools: List[str] = [] + for tool in root_agent.get("tools") or []: + if not isinstance(tool, dict): + continue + name = tool.get("origin_name") or tool.get("name") + if name: + tools.append(str(name)) + return tools + + +def _serialize_created_at(create_time: Any) -> Optional[str]: + """Serialize DB create_time to an ISO string for API consumers.""" + if create_time is None: + return None + if hasattr(create_time, "isoformat"): + return create_time.isoformat() + return str(create_time) + + +def get_agent_repository_listing_detail_impl( + agent_repository_id: int, + tenant_id: str, +) -> Dict[str, Any]: + """Load a repository listing and return a detail payload for the UI.""" + record = get_agent_repository_by_id_and_publisher( + agent_repository_id, + tenant_id, + ) + if not record: + raise ValueError("Repository listing not found") + + root_agent = _extract_root_agent_from_snapshot(record.get("agent_info_json")) + + return { + "agent_repository_id": record.get("agent_repository_id"), + "agent_id": record.get("agent_id"), + "name": record.get("name"), + "display_name": record.get("display_name"), + "description": record.get("description"), + "author": record.get("author"), + "submitted_by": record.get("submitted_by"), + "icon": record.get("icon"), + "status": record.get("status"), + "version_label": record.get("version_name"), + "downloads": record.get("downloads") or 0, + "created_at": _serialize_created_at(record.get("create_time")), + "model_name": root_agent.get("model_name"), + "duty_prompt": root_agent.get("duty_prompt"), + "tools": _extract_tool_names(root_agent), + } + + +def _get_user_role(user_id: str) -> str: + """Resolve user role from user_tenant_t; default to USER when unset.""" + user_tenant = get_user_tenant_by_user_id(user_id) + if not user_tenant: + return "USER" + return str(user_tenant.get("user_role") or "USER") + + +def _validate_create_listing_permission( + *, + user_id: str, + agent_info: Dict[str, Any], +) -> None: + """Only ADMIN, or DEV whose email matches agent.author, may share to marketplace.""" + user_role = _get_user_role(user_id) + if user_role == "ADMIN": + return + if user_role == "DEV": + user_tenant = get_user_tenant_by_user_id(user_id) or {} + user_email = str(user_tenant.get("user_email") or "").strip() + agent_author = str(agent_info.get("author") or "").strip() + if user_email and agent_author and user_email.lower() == agent_author.lower(): + return + raise UnauthorizedError("Not authorized to create repository listing") + raise UnauthorizedError( + f"User role {user_role} not authorized to create repository listing" + ) + + +def _validate_repository_status_transition( + *, + user_role: str, + current_status: str, + new_status: str, + record: Dict[str, Any], + user_id: str, + tenant_id: str, +) -> Optional[Dict[str, str]]: + """Validate role, ownership, and allowed status transition. + + Returns publisher fields to update when not_shared -> pending_review, + otherwise None. + """ + transition = (current_status, new_status) + + if user_role == "SU": + if transition not in _SU_STATUS_TRANSITIONS: + raise ValueError( + f"Invalid status transition from '{current_status}' to '{new_status}'" + ) + return None + + if user_role in ("ADMIN", "DEV"): + if record.get("publisher_tenant_id") != tenant_id: + raise UnauthorizedError( + "Not authorized to update this repository listing" + ) + if user_role == "DEV" and record.get("publisher_user_id") != user_id: + raise UnauthorizedError( + "Not authorized to update this repository listing" + ) + if ( + user_role == "ADMIN" + and transition in _ADMIN_REVIEW_STATUS_TRANSITIONS + ): + return None + if transition not in _PUBLISHER_STATUS_TRANSITIONS: + raise ValueError( + f"Invalid status transition from '{current_status}' to '{new_status}'" + ) + if transition in _PUBLISHER_RESUBMIT_TRANSITIONS: + return { + "publisher_tenant_id": tenant_id, + "publisher_user_id": user_id, + } + return None + + raise UnauthorizedError( + f"User role {user_role} not authorized to update repository status" + ) + + def update_agent_repository_status_impl( *, agent_repository_id: int, status: str, user_id: str, + tenant_id: str, ) -> Dict[str, Any]: """Update a repository listing status by primary key.""" if status not in VALID_REPOSITORY_STATUSES: @@ -77,19 +490,60 @@ def update_agent_repository_status_impl( f"{', '.join(sorted(VALID_REPOSITORY_STATUSES))}" ) - record = get_agent_repository_by_id(agent_repository_id) + record = get_agent_repository_by_id_and_publisher( + agent_repository_id, + tenant_id, + ) if not record: raise ValueError("Repository listing not found") + current_status = record.get("status") + publisher_updates: Optional[Dict[str, str]] = None + submitted_by: Optional[str] = None + if current_status != status: + user_role = _get_user_role(user_id) + publisher_updates = _validate_repository_status_transition( + user_role=user_role, + current_status=current_status, + new_status=status, + record=record, + user_id=user_id, + tenant_id=tenant_id, + ) + if status == STATUS_PENDING_REVIEW: + submitted_by = _resolve_submitter_email(user_id) + rows_affected = update_agent_repository_status_by_id( repository_id=agent_repository_id, status=status, user_id=user_id, + filter_publisher_tenant_id=tenant_id, + publisher_tenant_id=( + publisher_updates["publisher_tenant_id"] + if publisher_updates + else None + ), + publisher_user_id=( + publisher_updates["publisher_user_id"] + if publisher_updates + else None + ), + submitted_by=submitted_by, ) if rows_affected == 0: raise ValueError("Repository listing not found") - updated = get_agent_repository_by_id(agent_repository_id) + _reset_repository_peer_statuses( + agent_repository_id=agent_repository_id, + agent_id=record["agent_id"], + status=status, + publisher_tenant_id=tenant_id, + ) + + updated = get_agent_repository_by_id_and_publisher( + agent_repository_id, + tenant_id, + ) if not updated: raise ValueError("Failed to load repository listing after update") return _to_summary_item(updated) @@ -105,12 +559,15 @@ def _to_list_item(record: Dict[str, Any]) -> Dict[str, Any]: "display_name": record.get("display_name"), "description": record.get("description"), "author": record.get("author"), + "submitted_by": record.get("submitted_by"), "category_id": record.get("category_id"), "tags": record.get("tags") or [], "tool_count": record.get("tool_count"), - "version_label": record.get("version_label"), + "version_label": record.get("version_name"), + "icon": record.get("icon"), + "downloads": record.get("downloads") or 0, "status": record.get("status"), - "source_version_no": record.get("source_version_no"), + "version_no": record.get("version_no"), "publisher_tenant_id": record.get("publisher_tenant_id"), "created_at": record.get("create_time"), "updated_at": record.get("update_time"), @@ -136,7 +593,7 @@ def _validate_create_payload(repository_data: Dict[str, Any]) -> None: """Validate required fields before inserting a repository listing.""" required_fields = ( "agent_id", - "source_version_no", + "version_no", "name", "agent_info_json", ) @@ -156,17 +613,7 @@ def _validate_create_payload(repository_data: Dict[str, Any]) -> None: if key not in agent_info_json: raise ValueError(f"agent_info_json must contain '{key}'") - -def _validate_agent_info_json_shareable(agent_info_json: dict) -> None: - """Reject marketplace share when any agent in the tree belongs to ASSET_OWNER tenant.""" - agent_info_map = agent_info_json.get("agent_info") - if not isinstance(agent_info_map, dict): - return - for entry in agent_info_map.values(): - if not isinstance(entry, dict): - continue - if entry.get("tenant_id") == ASSET_OWNER_TENANT_ID: - raise ValueError("租户管理员智能体无法共享") + _validate_card_fields(repository_data) async def _build_agent_info_json( @@ -199,60 +646,82 @@ async def _build_repository_data_from_agent( tenant_id: str, user_id: str, version_no: int, + *, + card_fields: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Build a repository upsert payload from a published agent version snapshot.""" agent_info = search_agent_info_by_agent_id(agent_id, tenant_id, version_no) + _validate_create_listing_permission(user_id=user_id, agent_info=agent_info) agent_info_json = await _build_agent_info_json( agent_id=agent_id, tenant_id=tenant_id, user_id=user_id, version_no=version_no, ) - _validate_agent_info_json_shareable(agent_info_json) version_meta = search_version_by_version_no(agent_id, tenant_id, version_no) - version_label = ( + version_name = ( version_meta.get("version_name") if version_meta and version_meta.get("version_name") else f"v{version_no}" ) - return { + repository_data: Dict[str, Any] = { "agent_id": agent_id, - "source_version_no": version_no, + "version_no": version_no, "name": agent_info["name"], "display_name": agent_info.get("display_name"), "description": agent_info.get("description"), "author": agent_info.get("author"), - "version_label": version_label, + "submitted_by": _resolve_submitter_email(user_id), + "version_name": version_name, "agent_info_json": agent_info_json, "status": STATUS_PENDING_REVIEW, } + if card_fields: + for key in ("icon", "downloads", "category_id", "tool_count"): + if key in card_fields and card_fields[key] is not None: + repository_data[key] = card_fields[key] + if "tags" in card_fields and card_fields["tags"] is not None: + repository_data["tags"] = _normalize_listing_tags(card_fields["tags"]) + + return repository_data + async def create_agent_repository_listing_impl( agent_id: int, tenant_id: str, user_id: str, version_no: int, + *, + card_fields: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Create or update a repository listing from a published agent version. Loads agent metadata and builds agent_info_json via the export pipeline, then inserts or updates the marketplace table. - When a listing for the same agent_id already exists, snapshot fields are - updated via update_agent_repository_by_id. + When a listing for the same agent version already exists, snapshot fields + are updated via update_agent_repository_by_id. """ if version_no < 0: raise ValueError("version_no must be >= 0") repository_data = await _build_repository_data_from_agent( - agent_id, tenant_id, user_id, version_no + agent_id, + tenant_id, + user_id, + version_no, + card_fields=card_fields, ) _validate_create_payload(repository_data) - existing = get_agent_repository_by_agent_id(agent_id) + existing = get_agent_repository_by_agent_id( + agent_id, + version_no, + publisher_tenant_id=tenant_id, + ) if not existing: repository_id = insert_agent_repository_record( repository_data=repository_data, @@ -277,18 +746,31 @@ async def create_agent_repository_listing_impl( raise ValueError("Failed to update repository listing") is_updated = True - record = get_agent_repository_by_id(repository_id) + record = get_agent_repository_by_id_and_publisher( + repository_id, + tenant_id, + ) if not record: raise ValueError("Failed to load repository listing after write") + _reset_repository_peer_statuses( + agent_repository_id=repository_id, + agent_id=agent_id, + status=repository_data["status"], + publisher_tenant_id=tenant_id, + ) return _to_detail_item(record, is_updated=is_updated) async def import_agent_from_repository_impl( agent_repository_id: int, + tenant_id: str, authorization: str, ) -> Dict[int, int]: """Import an agent tree from a marketplace repository listing into the current tenant.""" - record = get_agent_repository_by_id(agent_repository_id) + record = get_agent_repository_by_id_and_publisher( + agent_repository_id, + tenant_id, + ) if not record: raise ValueError("Repository listing not found") diff --git a/backend/services/agent_service.py b/backend/services/agent_service.py index 643d1995e..c6a1ae80c 100644 --- a/backend/services/agent_service.py +++ b/backend/services/agent_service.py @@ -1109,6 +1109,7 @@ async def get_creating_sub_agent_info_impl(authorization: str = Header(None)): "model_name": agent_info["model_name"], "model_id": agent_info.get("model_id"), "max_steps": agent_info["max_steps"], + "requested_output_tokens": agent_info.get("requested_output_tokens"), "business_description": agent_info["business_description"], "duty_prompt": agent_info.get("duty_prompt"), "constraint_prompt": agent_info.get("constraint_prompt"), @@ -1116,12 +1117,52 @@ async def get_creating_sub_agent_info_impl(authorization: str = Header(None)): "sub_agent_id_list": query_sub_agents_id_list(main_agent_id=sub_agent_id, tenant_id=tenant_id)} +def _validate_requested_output_tokens_for_agent( + request: AgentInfoRequest, + tenant_id: str, +) -> None: + requested_output_tokens = request.requested_output_tokens + if requested_output_tokens is None: + return + + model_id = request.model_id + if model_id is None and request.agent_id is not None: + try: + existing_agent = search_agent_info_by_agent_id( + agent_id=request.agent_id, + tenant_id=tenant_id, + version_no=request.version_no, + ) + model_id = existing_agent.get("model_id") + except Exception as exc: + logger.warning( + "Could not resolve existing agent model for requested_output_tokens validation: %s", + exc, + ) + + if model_id is None: + return + + model_info = get_model_by_model_id(model_id, tenant_id=tenant_id) + max_output_tokens = model_info.get("max_output_tokens") if model_info else None + if max_output_tokens is not None and requested_output_tokens > max_output_tokens: + raise AppException( + ErrorCode.COMMON_PARAMETER_INVALID, + ( + "requested_output_tokens cannot exceed the selected model " + f"max_output_tokens ({max_output_tokens})" + ), + ) + + async def update_agent_info_impl(request: AgentInfoRequest, authorization: str = Header(None)): user_id, tenant_id, _ = get_current_user_info(authorization) if request.example_questions is not None and len(request.example_questions) > 6: raise AppException(ErrorCode.COMMON_PARAMETER_INVALID, "example_questions cannot exceed 6 items") + _validate_requested_output_tokens_for_agent(request, tenant_id) + prompt_template_id, prompt_template_name = get_prompt_template_summary( template_id=request.prompt_template_id, tenant_id=tenant_id, @@ -1147,6 +1188,7 @@ async def update_agent_info_impl(request: AgentInfoRequest, authorization: str = "prompt_template_id": prompt_template_id, "prompt_template_name": prompt_template_name, "max_steps": request.max_steps, + "requested_output_tokens": request.requested_output_tokens, "provide_run_summary": request.provide_run_summary, "verification_config": request.verification_config, "duty_prompt": request.duty_prompt, @@ -1199,7 +1241,9 @@ async def update_agent_info_impl(request: AgentInfoRequest, authorization: str = if inst.get("tool_id") == tool_id), None ) - params = (existing_instance or {}).get("params", {}) + # Safely get params, default to empty dict if None or not present + raw_params = (existing_instance or {}).get("params") + params = raw_params if raw_params is not None else {} create_or_update_tool_by_tool_info( tool_info=ToolInstanceInfoRequest( tool_id=tool_id, @@ -1673,6 +1717,7 @@ async def export_agent_by_agent_id( business_description=agent_info["business_description"], author=agent_info.get("author"), max_steps=agent_info["max_steps"], + requested_output_tokens=agent_info.get("requested_output_tokens"), provide_run_summary=agent_info["provide_run_summary"], verification_config=agent_info.get("verification_config"), duty_prompt=agent_info.get( @@ -1828,6 +1873,7 @@ async def import_agent_by_agent_id( "prompt_template_id": import_agent_info.prompt_template_id or SYSTEM_PROMPT_TEMPLATE_ID, "prompt_template_name": import_agent_info.prompt_template_name or SYSTEM_PROMPT_TEMPLATE_NAME, "max_steps": import_agent_info.max_steps, + "requested_output_tokens": import_agent_info.requested_output_tokens, "provide_run_summary": import_agent_info.provide_run_summary, "verification_config": getattr(import_agent_info, "verification_config", None), "duty_prompt": import_agent_info.duty_prompt, @@ -2197,6 +2243,7 @@ async def prepare_agent_run( is_debug=agent_request.is_debug, override_version_no=agent_request.version_no, override_model_id=agent_request.model_id, + requested_output_tokens=agent_request.requested_output_tokens, tool_params=agent_request.tool_params, ) diff --git a/backend/services/agent_version_service.py b/backend/services/agent_version_service.py index 8ed6e14d4..7bbcf606d 100644 --- a/backend/services/agent_version_service.py +++ b/backend/services/agent_version_service.py @@ -44,8 +44,6 @@ def _remove_audit_fields_for_insert(data: dict) -> None: """ data.pop('create_time', None) data.pop('update_time', None) - data.pop('created_by', None) - data.pop('updated_by', None) data.pop('delete_flag', None) @@ -90,6 +88,7 @@ def publish_version_impl( agent_snapshot.pop('version_no', None) agent_snapshot.pop('current_version_no', None) agent_snapshot['version_no'] = new_version_no + agent_snapshot['updated_by'] = user_id _remove_audit_fields_for_insert(agent_snapshot) # Insert agent snapshot @@ -100,6 +99,7 @@ def publish_version_impl( tool_snapshot = tool.copy() tool_snapshot.pop('version_no', None) tool_snapshot['version_no'] = new_version_no + tool_snapshot['updated_by'] = user_id _remove_audit_fields_for_insert(tool_snapshot) insert_tool_snapshot(tool_snapshot) @@ -115,6 +115,7 @@ def publish_version_impl( rel_snapshot.pop('version_no', None) rel_snapshot['version_no'] = new_version_no rel_snapshot['selected_agent_version_no'] = child_version + rel_snapshot['updated_by'] = user_id _remove_audit_fields_for_insert(rel_snapshot) insert_relation_snapshot(rel_snapshot) @@ -131,6 +132,7 @@ def publish_version_impl( skill_snapshot = skill.copy() skill_snapshot.pop('version_no', None) skill_snapshot['version_no'] = new_version_no + skill_snapshot['updated_by'] = user_id _remove_audit_fields_for_insert(skill_snapshot) insert_skill_snapshot(skill_snapshot) diff --git a/backend/services/aidp_service.py b/backend/services/aidp_service.py new file mode 100644 index 000000000..d92f770c6 --- /dev/null +++ b/backend/services/aidp_service.py @@ -0,0 +1,249 @@ +""" +AIDP Service Layer +Handles API calls to AIDP for paginated knowledge base listing. +""" +import logging +from typing import Any, Dict, List +from urllib.parse import urljoin + +import httpx + +from consts.error_code import ErrorCode +from consts.exceptions import AppException +from nexent.utils.http_client_manager import http_client_manager + +logger = logging.getLogger("aidp_service") + +_LIST_PATH = "/KnowledgeBase/Tenants/aidp/KnowledgeBases" + + +def _validate_params(server_url: str, api_key: str) -> str: + """Validate parameters and return normalized base URL.""" + if not server_url or not isinstance(server_url, str): + raise AppException( + ErrorCode.AIDP_CONFIG_INVALID, + "AIDP server_url is required and must be a non-empty string", + ) + if not server_url.startswith(("http://", "https://")): + raise AppException( + ErrorCode.AIDP_CONFIG_INVALID, + "AIDP server_url must start with http:// or https://", + ) + if not api_key or not isinstance(api_key, str): + raise AppException( + ErrorCode.AIDP_CONFIG_INVALID, + "AIDP api_key is required and must be a non-empty string", + ) + return server_url.rstrip("/") + + +def fetch_aidp_knowledge_bases_impl( + server_url: str, + api_key: str, + page: int = 1, + page_size: int = 10, +) -> Dict[str, Any]: + """Fetch a single page from AIDP API (simple passthrough).""" + normalized_url = _validate_params(server_url, api_key) + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + list_path = f"{_LIST_PATH}?page={page}&page_size={page_size}" + list_url = urljoin(f"{normalized_url}/", list_path) + logger.info("Fetching AIDP knowledge bases from %s", list_url) + + try: + client = http_client_manager.get_sync_client( + base_url=normalized_url, + timeout=60.0, + verify_ssl=False, + ) + response = client.get(list_url, headers=headers) + response.raise_for_status() + result = response.json() + if not isinstance(result, dict): + raise AppException( + ErrorCode.AIDP_SERVICE_ERROR, + "Unexpected AIDP knowledge base response format", + ) + return _normalize_response(result) + except httpx.RequestError as e: + logger.exception("AIDP request failed: %s", e) + raise AppException( + ErrorCode.AIDP_CONNECTION_ERROR, + f"AIDP API request failed: {str(e)}", + ) + except httpx.HTTPStatusError as e: + logger.exception( + "AIDP API HTTP error: %s, status_code: %s", + e, + e.response.status_code, + ) + if e.response.status_code in (401, 403): + raise AppException( + ErrorCode.AIDP_AUTH_ERROR, + f"AIDP authentication failed: {str(e)}", + ) + raise AppException( + ErrorCode.AIDP_SERVICE_ERROR, + f"AIDP API HTTP error {e.response.status_code}: {str(e)}", + ) + except ValueError as e: + logger.exception("Failed to parse AIDP API response: %s", e) + raise AppException( + ErrorCode.AIDP_SERVICE_ERROR, + f"Failed to parse AIDP API response: {str(e)}", + ) + + +def _normalize_response(raw: Dict[str, Any]) -> Dict[str, Any]: + """Map AIDP API response fields to the canonical {value, total_count, next_link} shape.""" + items = ( + raw.get("value") + if raw.get("value") is not None + else raw.get("data") + if raw.get("data") is not None + else raw.get("items") + if raw.get("items") is not None + else raw.get("knowledge_bases") + if raw.get("knowledge_bases") is not None + else [] + ) + total_keys = ("total_count", "total", "totalRecords", "count") + total = next((raw.get(k) for k in total_keys if raw.get(k) is not None), None) + next_link = raw.get("next_link") or raw.get("next") or None + return { + "value": items, + "total_count": total, + "next_link": next_link, + } + + +def _extract_tenant_from_url(url: str) -> str | None: + """Extract tenant ID from a URL like /KnowledgeBase/Tenants/{tenant}/KnowledgeBases.""" + import re + match = re.search(r"/Tenants/([^/]+)/", url) + return match.group(1) if match else None + + +def fetch_all_aidp_knowledge_bases_impl( + server_url: str, + api_key: str, +) -> Dict[str, Any]: + """Fetch all knowledge bases from AIDP by following next_link until exhausted. + + AIDP does not return a true total count, so we follow next_link pages + until there is no next_link left. We also detect the real tenant ID + from the first response's next_link (AIDP embeds it there) and use it + for any manual page construction needed. + """ + normalized_url = _validate_params(server_url, api_key) + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + try: + client = http_client_manager.get_sync_client( + base_url=normalized_url, + timeout=120.0, + verify_ssl=False, + ) + + all_items: List[Any] = [] + current_page = 1 + max_pages = 1000 + page_size = 100 + detected_tenant: str | None = None + + # Build the first request URL using the known path pattern + first_path = f"{_LIST_PATH}?page=1&page_size={page_size}" + current_url: str | None = urljoin(f"{normalized_url}/", first_path) + + while current_page <= max_pages and current_url: + logger.info( + "Fetching AIDP KBs — page %d from %s", + current_page, + current_url, + ) + + response = client.get(current_url, headers=headers) + response.raise_for_status() + result = response.json() + if not isinstance(result, dict): + raise AppException( + ErrorCode.AIDP_SERVICE_ERROR, + "Unexpected AIDP knowledge base response format", + ) + + page_items = ( + result.get("value") + if result.get("value") is not None + else result.get("data") + if result.get("data") is not None + else result.get("items") + if result.get("items") is not None + else result.get("knowledge_bases") + if result.get("knowledge_bases") is not None + else [] + ) + if not isinstance(page_items, list): + page_items = [] + + all_items.extend(page_items) + + # Detect real tenant from next_link on the first page + if current_page == 1 and detected_tenant is None: + raw_next = result.get("next_link") or result.get("next") or "" + detected_tenant = _extract_tenant_from_url(str(raw_next)) + if detected_tenant: + logger.info("Detected AIDP tenant: %s", detected_tenant) + + # Follow next_link if present, otherwise construct next page manually + raw_next = result.get("next_link") or result.get("next") or "" + next_url_str = str(raw_next).strip() + if next_url_str: + current_url = urljoin(normalized_url + "/", next_url_str) + current_page += 1 + else: + current_url = None + + total_count = len(all_items) + logger.info("AIDP KBs: accumulated %d total items (tenant=%s)", total_count, detected_tenant) + + return { + "value": all_items, + "total_count": total_count, + "next_link": None, + } + except httpx.RequestError as e: + logger.exception("AIDP request failed: %s", e) + raise AppException( + ErrorCode.AIDP_CONNECTION_ERROR, + f"AIDP API request failed: {str(e)}", + ) + except httpx.HTTPStatusError as e: + logger.exception( + "AIDP API HTTP error: %s, status_code: %s", + e, + e.response.status_code, + ) + if e.response.status_code in (401, 403): + raise AppException( + ErrorCode.AIDP_AUTH_ERROR, + f"AIDP authentication failed: {str(e)}", + ) + raise AppException( + ErrorCode.AIDP_SERVICE_ERROR, + f"AIDP API HTTP error {e.response.status_code}: {str(e)}", + ) + except ValueError as e: + logger.exception("Failed to parse AIDP API response: %s", e) + raise AppException( + ErrorCode.AIDP_SERVICE_ERROR, + f"Failed to parse AIDP API response: {str(e)}", + ) diff --git a/backend/services/conversation_management_service.py b/backend/services/conversation_management_service.py index 0b7345461..12edea7d5 100644 --- a/backend/services/conversation_management_service.py +++ b/backend/services/conversation_management_service.py @@ -127,7 +127,15 @@ def save_message(request: MessageRequest, user_id: str, tenant_id: str): # Parse image URL list content_json = json.loads(unit_content) if isinstance(content_json, dict) and 'images_url' in content_json: + # Deduplicate image URLs before saving + seen_urls = set() + unique_urls = [] for image_url in content_json['images_url']: + if image_url not in seen_urls: + seen_urls.add(image_url) + unique_urls.append(image_url) + # Also deduplicate against any URLs already saved in this same message + for image_url in unique_urls: image_data = {'message_id': message_id, 'conversation_id': conversation_id, 'image_url': image_url} create_source_image(image_data) @@ -448,13 +456,15 @@ def get_conversation_history_service(conversation_id: int, user_id: str) -> List search_by_message[message_id] = [] search_by_message[message_id].append(search_item) - # Collect image content - grouped by message_id + # Collect image content - grouped by message_id, with URL deduplication image_by_message = {} for record in history_data['image_records']: message_id = record['message_id'] if message_id not in image_by_message: image_by_message[message_id] = [] - image_by_message[message_id].append(record['image_url']) + # Only add if not already present (by URL) + if record['image_url'] not in image_by_message[message_id]: + image_by_message[message_id].append(record['image_url']) # Sort by message index and build final message list, including images and search content messages = [] diff --git a/backend/services/file_management_service.py b/backend/services/file_management_service.py index 585669c0c..64f7ac486 100644 --- a/backend/services/file_management_service.py +++ b/backend/services/file_management_service.py @@ -33,6 +33,7 @@ list_files, upload_fileobj, ) +from database.model_management_db import get_model_by_model_id from services.vectordatabase_service import ElasticSearchService, get_vector_db_core from utils.config_utils import tenant_config_manager, get_model_name_from_config from utils.file_management_utils import save_upload_file @@ -448,20 +449,39 @@ async def list_files_impl(prefix: str, limit: Optional[int] = None): return files -def get_llm_model(tenant_id: str): - # Get the tenant config - main_model_config = tenant_config_manager.get_model_config( - key=MODEL_CONFIG_MAPPING["llm"], tenant_id=tenant_id) +def get_llm_model(tenant_id: str, model_id: Optional[int] = None): + if model_id: + main_model_config = get_model_by_model_id(int(model_id), tenant_id) + if not main_model_config: + raise ValueError(f"Model not found: {model_id}") + if main_model_config.get("model_type") != "llm": + raise ValueError(f"Selected model {model_id} is not an LLM model") + else: + # Get the tenant config + main_model_config = tenant_config_manager.get_model_config( + key=MODEL_CONFIG_MAPPING["llm"], tenant_id=tenant_id) timeout_seconds = main_model_config.get( "timeout_seconds") if main_model_config else None + + resolved_model_name = get_model_name_from_config(main_model_config) + + logger.info( + "Using LLM model for analyze_text_file: model_id=%s, display_name=%s, model_name=%s", + model_id, + main_model_config.get("display_name") if main_model_config else None, + resolved_model_name + ) + long_text_to_text_model = OpenAILongContextModel( observer=MessageObserver(), - model_id=get_model_name_from_config(main_model_config), + model_id=resolved_model_name, api_base=main_model_config.get("base_url"), api_key=main_model_config.get("api_key"), max_context_tokens=main_model_config.get("max_tokens"), ssl_verify=main_model_config.get("ssl_verify", True), timeout_seconds=timeout_seconds, + model_factory=main_model_config.get("model_factory"), + display_name=main_model_config.get("display_name"), ) return long_text_to_text_model diff --git a/backend/services/image_service.py b/backend/services/image_service.py index 8a924e9cc..76790dc23 100644 --- a/backend/services/image_service.py +++ b/backend/services/image_service.py @@ -1,10 +1,16 @@ +import base64 +import ipaddress import logging +import socket from http import HTTPStatus +from typing import Optional +from urllib.parse import urlparse, urlunparse import aiohttp from consts.const import DATA_PROCESS_SERVICE from consts.const import MODEL_CONFIG_MAPPING +from database.model_management_db import get_model_by_model_id from utils.config_utils import tenant_config_manager, get_model_name_from_config from nexent import MessageObserver @@ -13,7 +19,119 @@ logger = logging.getLogger("image_service") +def _validate_loopback_url(decoded_url: str) -> str | None: + """Validate that ``decoded_url`` is a genuine loopback URL and return a + rewritten URL whose host is a literal IPv4 loopback address, or ``None`` + when the input is not safe to fetch directly. + + This is a defense-in-depth check for the fast-path that bypasses the + data-processing service. The fast-path is only intended for loopback + images (e.g. served by an in-process component), so we must verify: + + * The scheme is ``http`` or ``https``. + * The hostname resolves to one or more IPv4 addresses, and **every** + resolved address falls inside the standard IPv4 loopback range + ``127.0.0.0/8``. Mixed results are rejected to prevent an attacker + from racing DNS to a private address. + * The URL is rewritten so the host portion is a literal loopback IP. + This both (a) removes the user-controlled hostname from the final + request URL that ``aiohttp`` issues, and (b) blocks DNS rebinding + attacks where the hostname is re-resolved to a private address + between validation and the actual ``GET``. + """ + try: + parsed = urlparse(decoded_url) + except Exception: + return None + + if parsed.scheme not in {"http", "https"}: + return None + + hostname = parsed.hostname + if not hostname: + return None + + try: + resolved_infos = socket.getaddrinfo(hostname, None) + except socket.gaierror: + return None + + if not resolved_infos: + return None + + safe_addresses: list[str] = [] + for info in resolved_infos: + sockaddr = info[4] + candidate = sockaddr[0] + try: + ip = ipaddress.ip_address(candidate) + except ValueError: + return None + if ip.version != 4 or not ip.is_loopback: + return None + safe_addresses.append(candidate) + + # Prefer the literal 127.0.0.1 to keep the rewritten URL stable when + # the hostname resolves to multiple loopback aliases. + chosen_ip = ( + "127.0.0.1" if "127.0.0.1" in safe_addresses else safe_addresses[0] + ) + + port = parsed.port + netloc = f"{chosen_ip}:{port}" if port is not None else chosen_ip + + return urlunparse( + ( + parsed.scheme, + netloc, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + ) + ) + + +async def _fetch_image_directly(safe_url: str): + """Fetch an image from a previously validated loopback URL. + + ``safe_url`` MUST be the output of :func:`_validate_loopback_url` so that + it contains a literal loopback IPv4 address and is no longer + user-controlled. Redirects are disabled and ``trust_env`` is off to + ensure the request never leaks to a private/external host through + proxy variables or HTTP 30x responses. + """ + timeout = aiohttp.ClientTimeout(total=10) + async with aiohttp.ClientSession( + timeout=timeout, trust_env=False + ) as session: + async with session.get(safe_url, allow_redirects=False) as response: + if response.status != HTTPStatus.OK: + error_text = await response.text() + logger.error( + "Failed to fetch loopback image directly: %s", error_text + ) + return {"success": False, "error": "Failed to fetch image"} + + content = await response.read() + content_type = response.headers.get("Content-Type", "image/jpeg") + return { + "success": True, + "base64": base64.b64encode(content).decode("utf-8"), + "content_type": content_type, + } + + async def proxy_image_impl(decoded_url: str): + # Fast path: only for loopback URLs, fetch directly. This avoids an + # extra hop through the data-processing service for local images. For + # any other URL (including all external/knowledge-base images such as + # AIDP), fall back to the data-processing service proxy, which is the + # existing safe path that CodeQL does not flag. + safe_url = _validate_loopback_url(decoded_url) + if safe_url is not None: + return await _fetch_image_directly(safe_url) + # Create session to call the data processing service async with aiohttp.ClientSession() as session: # Call the data processing service to load the image @@ -30,14 +148,19 @@ async def proxy_image_impl(decoded_url: str): return result -def get_vlm_model(tenant_id: str): - """Return the configured image understanding model for AnalyzeImageTool. +def _get_model_config_by_id(tenant_id, model_id, expected_model_type): + if not model_id: + return None - The first multimodal model slot is still stored under MODEL_CONFIG_MAPPING["vlm"] - for compatibility, but it is the user-facing image understanding configuration. - """ - vlm_model_config = tenant_config_manager.get_model_config( - key=MODEL_CONFIG_MAPPING["vlm"], tenant_id=tenant_id) + model_config = get_model_by_model_id(int(model_id), tenant_id) + if not model_config: + raise ValueError(f"Model not found: {model_id}") + if model_config.get("model_type") != expected_model_type: + raise ValueError(f"Selected model {model_id} is not a {expected_model_type} model") + return model_config + + +def _build_vlm_model(vlm_model_config): if not vlm_model_config: return None return OpenAIVLModel( @@ -51,28 +174,34 @@ def get_vlm_model(tenant_id: str): frequency_penalty=0.5, max_tokens=512, ssl_verify=vlm_model_config.get("ssl_verify", True), + model_factory=vlm_model_config.get("model_factory"), + display_name=vlm_model_config.get("display_name"), ) +def get_vlm_model(tenant_id: str, model_id: Optional[int] = None): + """Return the configured image understanding model for AnalyzeImageTool. + + The first multimodal model slot is still stored under MODEL_CONFIG_MAPPING["vlm"] + for compatibility, but it is the user-facing image understanding configuration. + """ + if model_id: + vlm_model_config = _get_model_config_by_id(tenant_id, model_id, "vlm") + else: + vlm_model_config = tenant_config_manager.get_model_config( + key=MODEL_CONFIG_MAPPING["vlm"], tenant_id=tenant_id) + return _build_vlm_model(vlm_model_config) + + def get_image_understanding_model(tenant_id: str): return get_vlm_model(tenant_id=tenant_id) -def get_video_understanding_model(tenant_id: str): +def get_video_understanding_model(tenant_id: str, model_id: Optional[int] = None): """Return the configured video understanding model for multimodal tools.""" - vlm_model_config = tenant_config_manager.get_model_config( - key=MODEL_CONFIG_MAPPING["vlm3"], tenant_id=tenant_id) - if not vlm_model_config: - return None - return OpenAIVLModel( - observer=MessageObserver(), - model_id=get_model_name_from_config( - vlm_model_config) if vlm_model_config else "", - api_base=vlm_model_config.get("base_url", ""), - api_key=vlm_model_config.get("api_key", ""), - temperature=0.7, - top_p=0.7, - frequency_penalty=0.5, - max_tokens=512, - ssl_verify=vlm_model_config.get("ssl_verify", True), - ) + if model_id: + vlm_model_config = _get_model_config_by_id(tenant_id, model_id, "vlm3") + else: + vlm_model_config = tenant_config_manager.get_model_config( + key=MODEL_CONFIG_MAPPING["vlm3"], tenant_id=tenant_id) + return _build_vlm_model(vlm_model_config) diff --git a/backend/services/model_capacity_suggestion_service.py b/backend/services/model_capacity_suggestion_service.py new file mode 100644 index 000000000..723f0fd8e --- /dev/null +++ b/backend/services/model_capacity_suggestion_service.py @@ -0,0 +1,292 @@ +import re +from dataclasses import dataclass +from enum import Enum +from typing import Any, Mapping, Optional + +from consts.const import CAPACITY_SUGGESTION_ENABLED + + +ProfileKey = tuple[str, str] +CapabilityProfileLike = Any + + +class CapacitySuggestionMatchKind(str, Enum): + CATALOG_EXACT = "catalog_exact" + CATALOG_FUZZY = "catalog_fuzzy" + PROVIDER_DISCOVERY = "provider_discovery" + NONE = "none" + + +class CapacitySuggestionConfidence(str, Enum): + HIGH = "high" + MEDIUM = "medium" + LOW = "low" + + +@dataclass(frozen=True) +class CapacitySuggestionFields: + context_window_tokens: Optional[int] = None + max_input_tokens: Optional[int] = None + max_output_tokens: Optional[int] = None + default_output_reserve_tokens: Optional[int] = None + tokenizer_family: Optional[str] = None + + +@dataclass(frozen=True) +class CapacitySuggestionResult: + suggestions: Optional[CapacitySuggestionFields] + match_kind: CapacitySuggestionMatchKind + match_confidence: Optional[CapacitySuggestionConfidence] + match_explanation: str + suggested_provider: Optional[str] = None + canonical_model_name: Optional[str] = None + capability_profile_version: Optional[str] = None + capacity_source_on_accept: Optional[str] = None + + +# Substring patterns matched against the lower-cased base_url. Order matters: +# `in` returns the first hit, so place more-specific patterns before broader +# ones (e.g. `dashscope` before `aliyuncs`). Patterns mirror frontend +# PROVIDER_HINTS in `frontend/const/modelConfig.ts` so backend provider-by-URL +# detection stays consistent with the icon the user sees in the UI. +HOST_PROVIDER_PATTERNS = ( + ("dashscope", "dashscope"), + ("aliyuncs", "dashscope"), + ("siliconflow", "silicon"), + ("silicon", "silicon"), + ("modelengine", "modelengine"), + ("openai", "openai"), + ("deepseek", "deepseek"), + ("jina", "jina"), + ("tokenpony", "tokenpony"), + ("bytedance", "volcengine"), +) + +SUPPORTED_SUGGESTION_MODEL_TYPES = {"llm", "vlm", "vlm2", "vlm3"} + + +def pick_provider_from_base_url(base_url: Optional[str]) -> Optional[str]: + # Match the entire lower-cased base_url, mirroring the frontend + # detectProviderFromUrl helper. Substring `in` check, first hit wins. + if not base_url: + return None + + lowered = base_url.lower() + for pattern, provider in HOST_PROVIDER_PATTERNS: + if pattern in lowered: + return provider + return None + + +def _normalize_provider(provider: Optional[str]) -> Optional[str]: + if provider is None: + return None + normalized = provider.strip().lower() + if normalized in {"", "openai-api-compatible"}: + return None + if normalized == "siliconflow": + return "silicon" + return normalized + + +def normalize_model_name(model_name: str) -> str: + return re.sub(r"[-_./\s]+", "", model_name.strip().lower()) + + +def _normalize_catalog_exact_name(model_name: str) -> str: + return model_name.strip().lower() + + +def _profile_to_suggestion(profile: CapabilityProfileLike) -> CapacitySuggestionFields: + return CapacitySuggestionFields( + context_window_tokens=profile.context_window_tokens, + max_input_tokens=profile.max_input_tokens, + max_output_tokens=profile.max_output_tokens, + default_output_reserve_tokens=profile.default_output_reserve_tokens, + tokenizer_family=profile.tokenizer_family, + ) + + +def _result_from_profile( + provider: str, + model_name: str, + profile: CapabilityProfileLike, + match_kind: CapacitySuggestionMatchKind, +) -> CapacitySuggestionResult: + confidence = ( + CapacitySuggestionConfidence.HIGH + if match_kind == CapacitySuggestionMatchKind.CATALOG_EXACT + else CapacitySuggestionConfidence.MEDIUM + ) + return CapacitySuggestionResult( + suggestions=_profile_to_suggestion(profile), + match_kind=match_kind, + match_confidence=confidence, + match_explanation=f"Matched approved catalog profile {profile.capability_profile_version}", + suggested_provider=provider, + canonical_model_name=model_name, + capability_profile_version=profile.capability_profile_version, + capacity_source_on_accept="operator", + ) + + +def _none_result(explanation: str) -> CapacitySuggestionResult: + return CapacitySuggestionResult( + suggestions=None, + match_kind=CapacitySuggestionMatchKind.NONE, + match_confidence=None, + match_explanation=explanation, + ) + + +def _provider_catalog( + catalog: Mapping[ProfileKey, CapabilityProfileLike], + provider: str, +) -> dict[ProfileKey, CapabilityProfileLike]: + return { + (catalog_provider, catalog_model): profile + for (catalog_provider, catalog_model), profile in catalog.items() + if catalog_provider == provider + } + + +def _unique_final_segment_match( + model_name: str, + catalog: Mapping[ProfileKey, CapabilityProfileLike], + provider: str, +) -> Optional[tuple[ProfileKey, CapabilityProfileLike]]: + requested = normalize_model_name(model_name) + matches: list[tuple[ProfileKey, CapabilityProfileLike]] = [] + for key, profile in _provider_catalog(catalog, provider).items(): + catalog_model = key[1] + final_segment = catalog_model.split("/")[-1] + if normalize_model_name(final_segment) == requested: + matches.append((key, profile)) + + if len(matches) == 1: + return matches[0] + return None + + +def _fuzzy_catalog_match( + model_name: str, + catalog: Mapping[ProfileKey, CapabilityProfileLike], + provider: str, +) -> Optional[tuple[ProfileKey, CapabilityProfileLike]]: + requested = normalize_model_name(model_name) + matches: list[tuple[ProfileKey, CapabilityProfileLike]] = [] + for key, profile in _provider_catalog(catalog, provider).items(): + if normalize_model_name(key[1]) == requested: + matches.append((key, profile)) + + if len(matches) == 1: + return matches[0] + + return _unique_final_segment_match(model_name, catalog, provider) + + +def _unique_catalog_provider_for_model( + model_name: str, + catalog: Mapping[ProfileKey, CapabilityProfileLike], +) -> Optional[str]: + requested = normalize_model_name(model_name) + providers = { + provider + for provider, catalog_model in catalog.keys() + if normalize_model_name(catalog_model) == requested + or normalize_model_name(catalog_model.split("/")[-1]) == requested + } + if len(providers) == 1: + return next(iter(providers)) + return None + + +def pick_provider( + provider_hint: Optional[str], + base_url: Optional[str], + model_name: str, + catalog: Optional[Mapping[ProfileKey, CapabilityProfileLike]] = None, +) -> Optional[str]: + active_catalog = catalog if catalog is not None else _get_default_catalog() + explicit_provider = _normalize_provider(provider_hint) + if explicit_provider: + return explicit_provider + + inferred_provider = pick_provider_from_base_url(base_url) + if inferred_provider: + return inferred_provider + + return _unique_catalog_provider_for_model(model_name, active_catalog) + + +def _get_default_catalog() -> Mapping[ProfileKey, CapabilityProfileLike]: + from consts.capability_profiles import CATALOG + + return CATALOG + + +def suggest_capacity( + model_name: str, + base_url: Optional[str] = None, + provider_hint: Optional[str] = None, + model_type: Optional[str] = None, + api_key: Optional[str] = None, + catalog: Optional[Mapping[ProfileKey, CapabilityProfileLike]] = None, + enabled: bool = CAPACITY_SUGGESTION_ENABLED, +) -> CapacitySuggestionResult: + del api_key + + if not enabled: + return _none_result("Capacity suggestion is disabled") + + clean_model_name = (model_name or "").strip() + if not clean_model_name: + raise ValueError("model_name is required") + + if len(clean_model_name) > 512: + raise ValueError("model_name is too long") + + if model_type and model_type.lower() not in SUPPORTED_SUGGESTION_MODEL_TYPES: + return _none_result(f"Capacity suggestion is not supported for model_type={model_type}") + + active_catalog = catalog if catalog is not None else _get_default_catalog() + + provider = pick_provider(provider_hint, base_url, clean_model_name, active_catalog) + if not provider: + return _none_result("No provider candidate could be inferred") + + exact_key = (provider, clean_model_name) + exact_profile = active_catalog.get(exact_key) + if exact_profile: + return _result_from_profile( + provider, + clean_model_name, + exact_profile, + CapacitySuggestionMatchKind.CATALOG_EXACT, + ) + + normalized_exact_key = None + for catalog_key in _provider_catalog(active_catalog, provider).keys(): + if _normalize_catalog_exact_name(catalog_key[1]) == _normalize_catalog_exact_name(clean_model_name): + normalized_exact_key = catalog_key + break + + if normalized_exact_key: + return _result_from_profile( + normalized_exact_key[0], + normalized_exact_key[1], + active_catalog[normalized_exact_key], + CapacitySuggestionMatchKind.CATALOG_EXACT, + ) + + fuzzy_match = _fuzzy_catalog_match(clean_model_name, active_catalog, provider) + if fuzzy_match: + fuzzy_key, profile = fuzzy_match + return _result_from_profile( + fuzzy_key[0], + fuzzy_key[1], + profile, + CapacitySuggestionMatchKind.CATALOG_FUZZY, + ) + + return _none_result(f"No approved catalog profile matched provider={provider}, model={clean_model_name}") diff --git a/backend/services/model_health_service.py b/backend/services/model_health_service.py index 2dc276aeb..35fff2a23 100644 --- a/backend/services/model_health_service.py +++ b/backend/services/model_health_service.py @@ -38,13 +38,17 @@ def _normalize_embedding_url(base_url: str) -> str: def _infer_model_factory(model_type: str, base_url: str, current_factory: Optional[str] = None) -> Optional[str]: """Infer model_factory from base_url if not already set or is generic. - Currently handles: - - multi_embedding with dashscope URL -> "dashscope" - - embedding with dashscope URL -> "dashscope" (uses OpenAI-compatible endpoint) + Uses the shared W11 host map so embedding and LLM/VLM inference do not drift. """ - base_url_lower = base_url.lower() - if "dashscope" in base_url_lower: - return DASHSCOPE_MODEL_FACTORY + try: + from services.model_capacity_suggestion_service import pick_provider_from_base_url + + inferred_provider = pick_provider_from_base_url(base_url) + except Exception: + inferred_provider = DASHSCOPE_MODEL_FACTORY if "dashscope" in base_url.lower() else None + + if inferred_provider: + return inferred_provider return current_factory diff --git a/backend/services/model_management_service.py b/backend/services/model_management_service.py index 1511a9301..a8f28e133 100644 --- a/backend/services/model_management_service.py +++ b/backend/services/model_management_service.py @@ -1,7 +1,14 @@ import logging +import threading from typing import List, Dict, Any, Optional -from consts.const import LOCALHOST_IP, LOCALHOST_NAME, DOCKER_INTERNAL_HOST +from consts.const import ( + CAPACITY_SUGGESTION_ENABLED, + CAPACITY_VISIBILITY_ENABLED, + LOCALHOST_IP, + LOCALHOST_NAME, + DOCKER_INTERNAL_HOST, +) from consts.model import ModelConnectStatusEnum from consts.provider import ( ProviderEnum, @@ -26,6 +33,7 @@ get_provider_models, ) from services.model_health_service import embedding_dimension_check, _infer_model_factory +from services.model_capacity_suggestion_service import CapacitySuggestionMatchKind, suggest_capacity from utils.model_name_utils import ( add_repo_to_name, split_repo_name, @@ -38,6 +46,59 @@ logger = logging.getLogger("model_management_service") INDEPENDENT_MULTIMODAL_MODEL_TYPES = {"vlm", "vlm2", "vlm3"} +CAPACITY_COVERAGE_MODEL_TYPES = {"llm", "vlm", "vlm2", "vlm3"} + + +# OpenTelemetry counter for silent catalog-matcher failures during the +# capacity-coverage scan. The matcher is called per row so we cannot raise -- +# but the silent fallback to suggestion_available=False would hide a corrupt +# catalog entry that turns every "available" hint into "false" across a whole +# tenant. The counter gives staging/CI a single number to watch. +# +# Guarded the same way as the SDK monitor module: if OpenTelemetry is not +# installed (some deployments run without it), the counter is None and the +# increment becomes a no-op. +try: + from opentelemetry import metrics as _otel_metrics + + _capacity_suggestion_meter = _otel_metrics.get_meter(__name__) + _capacity_suggestion_coverage_errors_total = _capacity_suggestion_meter.create_counter( + name="model_capacity_suggestion_coverage_errors_total", + description=( + "Count of catalog-matcher exceptions raised while computing the " + "per-row `suggestion_available` flag in /model/capacity-coverage. " + "Non-zero means catalog data or matcher logic is broken; " + "operators see every row as suggestion_available=False." + ), + unit="errors", + ) +except Exception: # pragma: no cover - OTel is optional at runtime + _capacity_suggestion_coverage_errors_total = None + + +# Per-process dedup for the warning log emitted when the catalog-matcher +# raises during /capacity-coverage. The OTel counter still increments per +# failure (no monitoring impact); only the log line is deduped, so a global +# catalog bug surfaces once per (model_id, error_type) instead of flooding +# logs on every endpoint call. Same pattern as +# `_warn_missing_capacity_once` in `backend/agents/create_agent_info.py`. +_CAPACITY_SUGGESTION_ERROR_EMITTED: set = set() +_CAPACITY_SUGGESTION_ERROR_LOCK = threading.Lock() + + +def _record_capacity_coverage_error(model_id: Optional[Any], exc: Exception) -> None: + if _capacity_suggestion_coverage_errors_total is None: + return + try: + _capacity_suggestion_coverage_errors_total.add( + 1, + { + "model_id": str(model_id) if model_id is not None else "unknown", + "error_type": type(exc).__name__, + }, + ) + except Exception: # pragma: no cover - never break coverage for telemetry + pass def _has_display_name_conflict(existing_models: List[Dict[str, Any]], model_type: Optional[str]) -> bool: @@ -55,6 +116,104 @@ def _has_display_name_conflict(existing_models: List[Dict[str, Any]], model_type return True +def _coerce_legacy_max_tokens_alias(model_data: Dict[str, Any]) -> None: + """Keep the deprecated `max_tokens` column in lockstep with `max_output_tokens`. + + W1 step 7 deprecates `max_tokens` as the LLM/VLM output-cap alias of + `max_output_tokens`. Legacy clients that still write `max_tokens` + independently let the two columns diverge in the DB; that divergence + later surfaces at the W2 dispatch boundary as + `CallerMaxTokensOverrideForbidden` because the SDK auto-fills + `max_tokens` from the model record while the W2 snapshot computes its + output cap from `max_output_tokens`. + + Defense in depth at the service layer: when a caller sends a non-None + `max_output_tokens`, force `max_tokens` to mirror it. Embedding rows are + exempt because they repurpose `max_tokens` as the vector dimension. + """ + max_output = model_data.get("max_output_tokens") + if max_output is None: + return + if model_data.get("model_type") in ("embedding", "multi_embedding"): + return + model_data["max_tokens"] = max_output + + +def _is_bare_capacity_model(model: Dict[str, Any]) -> bool: + return model.get("context_window_tokens") is None or model.get("max_output_tokens") is None + + +def _capacity_suggestion_available(model: Dict[str, Any]) -> bool: + if not CAPACITY_SUGGESTION_ENABLED: + return False + + try: + model_name = add_repo_to_name(model.get("model_repo", ""), model.get("model_name", "")) + result = suggest_capacity( + model_name=model_name, + base_url=model.get("base_url"), + provider_hint=model.get("model_factory"), + model_type=model.get("model_type"), + enabled=CAPACITY_SUGGESTION_ENABLED, + ) + return result.match_kind != CapacitySuggestionMatchKind.NONE + except Exception as exc: + # A catalog-matcher exception must not break /capacity-coverage -- + # the endpoint scans every LLM/VLM row, and one bad row would make + # the whole tenant view explode. We fall back to False and emit a + # counter so a corrupt catalog is visible in metrics instead of + # silently turning every row into "no suggestion available". + dedup_key = (model.get("model_id"), type(exc).__name__) + should_log = False + with _CAPACITY_SUGGESTION_ERROR_LOCK: + if dedup_key not in _CAPACITY_SUGGESTION_ERROR_EMITTED: + _CAPACITY_SUGGESTION_ERROR_EMITTED.add(dedup_key) + should_log = True + if should_log: + logger.warning( + "Capacity coverage suggestion check failed for model_id=%s: %s " + "(per-process dedup; OTel counter still increments per failure)", + model.get("model_id"), + exc, + ) + _record_capacity_coverage_error(model.get("model_id"), exc) + return False + + +def get_capacity_coverage(tenant_id: str) -> Dict[str, Any]: + """Return bare-capacity LLM/VLM coverage for one tenant.""" + if not CAPACITY_VISIBILITY_ENABLED: + return { + "total_llm_vlm": 0, + "bare_count": 0, + "bare_models": [], + } + + records = get_model_records(None, tenant_id) + scoped_records = [ + model for model in records + if model.get("model_type") in CAPACITY_COVERAGE_MODEL_TYPES + ] + bare_models = [ + { + "model_id": model["model_id"], + "model_name": add_repo_to_name(model.get("model_repo", ""), model.get("model_name", "")), + "model_factory": model.get("model_factory"), + "model_type": model.get("model_type"), + "max_tokens": model.get("max_tokens"), + "suggestion_available": _capacity_suggestion_available(model), + } + for model in scoped_records + if _is_bare_capacity_model(model) + ] + + return { + "total_llm_vlm": len(scoped_records), + "bare_count": len(bare_models), + "bare_models": bare_models, + } + + async def create_model_for_tenant(user_id: str, tenant_id: str, model_data: Dict[str, Any]): """Create a single model record for the given tenant. @@ -93,6 +252,8 @@ async def create_model_for_tenant(user_id: str, tenant_id: str, model_data: Dict model_name=model_data.get("model_name", "") ) + _coerce_legacy_max_tokens_alias(model_data) + # Use NOT_DETECTED status as default model_data["connect_status"] = model_data.get( "connect_status") or ModelConnectStatusEnum.NOT_DETECTED.value @@ -208,9 +369,24 @@ async def batch_create_models_for_tenant(user_id: str, tenant_id: str, batch_pay for model in existing_model_list } - # Delete existing models not present + # Delete existing models not present. + # The membership key MUST match how existing_model_map (a few lines + # above) and the create-or-update branch (a few lines below) build + # their lookup key, otherwise the two halves disagree about what + # "the same model" means. Both of those use add_repo_to_name, which + # omits the slash when model_repo is empty. The naive + # `model_repo + "/" + model_name` here always prepends "/" for the + # empty-repo case (DashScope catalogs return bare names like + # "glm-4.7" and rows land with model_repo=""), so "/glm-4.7" never + # matched the catalog's "glm-4.7" entry -- every existing row was + # treated as "not in the incoming list" and silently soft-deleted on + # every batch_create. Use the same helper to keep both halves + # speaking the same language. for model in existing_model_list: - model_full_name = model["model_repo"] + "/" + model["model_name"] + model_full_name = add_repo_to_name( + model_repo=model["model_repo"], + model_name=model["model_name"], + ) if model_full_name not in model_list_ids: delete_model_record(model["model_id"], user_id, tenant_id) @@ -231,6 +407,31 @@ async def batch_create_models_for_tenant(user_id: str, tenant_id: str, batch_pay new_max_tokens = model.get("max_tokens") if new_max_tokens is not None and existing_max_tokens != new_max_tokens: update_data["max_tokens"] = new_max_tokens + # Same gap as prepare_model_dict had for the create branch: + # the batch refresh path only touched legacy max_tokens, so + # editing a row's capacity via batch-add (e.g. tweaking the + # top-level batch defaults and re-confirming) silently + # dropped the W1/W2 capacity updates. We mirror the + # operator-vs-candidate rule from prepare_model_dict here: + # only persist W1/W2 capacity when the payload is marked + # capacity_source="operator", so provider-discovered hints + # don't auto-overwrite an existing row on a refresh. + if model.get("capacity_source") == "operator": + for field in ( + "context_window_tokens", + "max_input_tokens", + "max_output_tokens", + "default_output_reserve_tokens", + "tokenizer_family", + "capability_profile_version", + ): + new_value = model.get(field) + if new_value is None: + continue + if existing_model.get(field) != new_value: + update_data[field] = new_value + if existing_model.get("capacity_source") != "operator": + update_data["capacity_source"] = "operator" if update_data: update_model_record(existing_model["model_id"], update_data, user_id) continue @@ -315,6 +516,16 @@ async def update_single_model_for_tenant( else: model_data["ssl_verify"] = True + # Carry model_type from the existing record so the legacy-alias + # coercion can distinguish LLM/VLM updates from embedding updates + # even when the caller payload omits model_type. We don't store the + # injected model_type back on model_data because the update path + # explicitly strips it later. + existing_model_type = existing_models[0].get("model_type") if existing_models else None + if model_data.get("max_output_tokens") is not None and \ + existing_model_type not in ("embedding", "multi_embedding"): + model_data["max_tokens"] = model_data["max_output_tokens"] + if has_multi_embedding: # Update both embedding and multi_embedding records for model in existing_models: @@ -343,6 +554,7 @@ async def batch_update_models_for_tenant(user_id: str, tenant_id: str, model_lis """Batch update models for a tenant by model_id or model_name.""" try: for model in model_list: + _coerce_legacy_max_tokens_alias(model) # Build update data excluding id fields update_data = {k: v for k, v in model.items() if k not in ["model_id", "model_name"]} @@ -571,4 +783,3 @@ async def list_models_for_admin( except Exception as e: logging.error(f"Failed to retrieve admin model list: {str(e)}") raise Exception(f"Failed to retrieve admin model list: {str(e)}") - diff --git a/backend/services/model_provider_service.py b/backend/services/model_provider_service.py index 1aa89fa3b..31867bedc 100644 --- a/backend/services/model_provider_service.py +++ b/backend/services/model_provider_service.py @@ -108,6 +108,35 @@ async def prepare_model_dict(provider: str, model: dict, model_url: str, model_a "max_tokens", 0) if not is_embedding_type else 0 timeout_seconds_value = 120 if not is_embedding_type else None + # W1/W2 capacity fields. The frontend batch-add resolves these in + # buildBatchModelData (row override -> top-level batch default) and + # sends them per row tagged with capacity_source. Two cases: + # - capacity_source="operator": the operator explicitly saved these + # values (top-level batch default panel or per-row gear modal). + # Persist them. Without this branch the ModelRequest defaults kick + # in (all None) and every freshly batch-created row lands with + # context_window_tokens=NULL, max_output_tokens=NULL even though + # the user filled the panel -- the glm-5.1/glm-5.2 incident. + # - capacity_source="provider_candidate" (or anything else): per the + # W1 design these are advisory UI hints surfaced from the catalog + # by _extract_capacity_hints. They are shown to the user as + # suggestions but not auto-persisted; only operator acceptance + # should write them. + is_operator_capacity = model.get("capacity_source") == "operator" + capacity_kwargs = ( + { + "context_window_tokens": model.get("context_window_tokens"), + "max_input_tokens": model.get("max_input_tokens"), + "max_output_tokens": model.get("max_output_tokens"), + "default_output_reserve_tokens": model.get("default_output_reserve_tokens"), + "tokenizer_family": model.get("tokenizer_family"), + "capacity_source": "operator", + "capability_profile_version": model.get("capability_profile_version"), + } + if is_operator_capacity + else {} + ) + model_obj = ModelRequest( model_factory=provider, model_name=model_name, @@ -118,7 +147,8 @@ async def prepare_model_dict(provider: str, model: dict, model_url: str, model_a expected_chunk_size=expected_chunk_size, maximum_chunk_size=maximum_chunk_size, chunk_batch=chunk_batch, - timeout_seconds=timeout_seconds_value + timeout_seconds=timeout_seconds_value, + **capacity_kwargs, ) model_dict = model_obj.model_dump() @@ -194,11 +224,20 @@ def merge_existing_model_attributes( if not model_list or not existing_model_list: return model_list - # Create a mapping table for existing models for quick lookup + # Create a mapping table for existing models for quick lookup. + # Use add_repo_to_name so the lookup key matches the format used by + # provider responses and downstream consumers. Naive `model_repo + "/" + + # model_name` prepends a leading slash when model_repo is empty + # (DashScope-style bare names like "glm-4.7" land with model_repo=""), + # so "/glm-4.7" never matches the catalog's "glm-4.7" entry and the + # merge silently no-ops -- the same wire-key bug fixed in + # batch_create_models_for_tenant's delete loop. existing_model_map = {} for existing_model in existing_model_list: - model_full_name = existing_model["model_repo"] + \ - "/" + existing_model["model_name"] + model_full_name = add_repo_to_name( + model_repo=existing_model["model_repo"], + model_name=existing_model["model_name"], + ) existing_model_map[model_full_name] = existing_model # Iterate through the model list, merge specified fields from existing models diff --git a/backend/services/northbound_service.py b/backend/services/northbound_service.py index c5493a551..a75b92ce0 100644 --- a/backend/services/northbound_service.py +++ b/backend/services/northbound_service.py @@ -133,7 +133,7 @@ def _normalize_northbound_attachments( tenant_id: str, ) -> Optional[List[Dict[str, Any]]]: """Convert northbound attachment references into internal minio_files objects. - + Supports two formats: 1. List of S3 URL strings (backward compatible): ["s3://nexent/...", "/nexent/...", "attachments/..."] 2. List of attachment objects (full metadata): [{"object_name": "...", "name": "...", ...}] diff --git a/backend/services/providers/base.py b/backend/services/providers/base.py index 4756bf6ad..0b0576765 100644 --- a/backend/services/providers/base.py +++ b/backend/services/providers/base.py @@ -1,12 +1,95 @@ import logging from abc import ABC, abstractmethod -from typing import Dict, List +from typing import Any, Dict, Iterable, List import aiohttp logger = logging.getLogger("model_provider") +_CONTEXT_WINDOW_KEYS = ( + "context_window_tokens", + "context_window", + "context_length", + "max_context_length", + "max_context_tokens", + "max_sequence_length", +) +_MAX_INPUT_KEYS = ("max_input_tokens", "input_token_limit", "max_prompt_tokens") +_MAX_OUTPUT_KEYS = ( + "max_output_tokens", + "output_token_limit", + "max_completion_tokens", + "max_tokens", +) +_OUTPUT_RESERVE_KEYS = ( + "default_output_reserve_tokens", + "default_output_reserve", + "output_reserve_tokens", +) +_TOKENIZER_KEYS = ("tokenizer_family", "tokenizer", "tokenizer_type") + + +def _positive_int(value: Any) -> int | None: + if isinstance(value, bool) or value is None: + return None + try: + parsed = int(value) + except (TypeError, ValueError): + return None + return parsed if parsed > 0 else None + + +def _candidate_dicts(raw: Dict, nested_keys: Iterable[str]) -> List[Dict]: + candidates = [raw] + for key in nested_keys: + value = raw.get(key) + if isinstance(value, dict): + candidates.append(value) + return candidates + + +def _first_positive_int(candidates: List[Dict], keys: tuple[str, ...]) -> int | None: + for candidate in candidates: + for key in keys: + value = _positive_int(candidate.get(key)) + if value is not None: + return value + return None + + +def _first_non_empty_str(candidates: List[Dict], keys: tuple[str, ...]) -> str | None: + for candidate in candidates: + for key in keys: + value = candidate.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _extract_capacity_hints_from_raw(raw: Dict, nested_keys: Iterable[str] = ()) -> Dict: + """Extract advisory provider-discovery capacity hints from one raw model row.""" + candidates = _candidate_dicts(raw, nested_keys) + hints = {} + for target_key, source_keys in ( + ("context_window_tokens", _CONTEXT_WINDOW_KEYS), + ("max_input_tokens", _MAX_INPUT_KEYS), + ("max_output_tokens", _MAX_OUTPUT_KEYS), + ("default_output_reserve_tokens", _OUTPUT_RESERVE_KEYS), + ): + value = _first_positive_int(candidates, source_keys) + if value is not None: + hints[target_key] = value + + tokenizer_family = _first_non_empty_str(candidates, _TOKENIZER_KEYS) + if tokenizer_family: + hints["tokenizer_family"] = tokenizer_family + + if hints: + hints["capacity_source"] = "provider_candidate" + return hints + + # ============================================================================= # Provider Error Handling Utilities # ============================================================================= diff --git a/backend/services/providers/dashscope_provider.py b/backend/services/providers/dashscope_provider.py index 497dcfe99..f78c57a3f 100644 --- a/backend/services/providers/dashscope_provider.py +++ b/backend/services/providers/dashscope_provider.py @@ -3,7 +3,11 @@ import asyncio from consts.const import DEFAULT_LLM_MAX_TOKENS from consts.provider import DASHSCOPE_GET_URL -from services.providers.base import AbstractModelProvider, _classify_provider_error +from services.providers.base import ( + AbstractModelProvider, + _classify_provider_error, + _extract_capacity_hints_from_raw, +) DASHSCOPE_IMAGE_GENERATION_KEYWORDS = ( @@ -33,6 +37,10 @@ DASHSCOPE_VIDEO_UNDERSTANDING_KEYWORDS = ("omni", "video-understanding", "video-ocr") +def _extract_capacity_hints(raw: Dict) -> Dict: + return _extract_capacity_hints_from_raw(raw, nested_keys=("inference_metadata",)) + + def _modality_set(value) -> set: if not value: return set() @@ -155,6 +163,7 @@ async def get_models(self, provider_config: Dict) -> List[Dict]: "model_type": "", "max_tokens": DEFAULT_LLM_MAX_TOKENS } + cleaned_model.update(_extract_capacity_hints(model_obj)) # 1. Embedding if 'embedding' in m_id.lower() or '向量' in desc: cleaned_model.update({"model_tag": "embedding", "model_type": "embedding"}) @@ -214,4 +223,3 @@ async def get_models(self, provider_config: Dict) -> List[Dict]: return [] except (httpx.HTTPStatusError, httpx.ConnectTimeout, httpx.ConnectError, Exception) as e: return _classify_provider_error("DashScope", exception=e) - diff --git a/backend/services/providers/modelengine_provider.py b/backend/services/providers/modelengine_provider.py index 276f84378..5b0e2b555 100644 --- a/backend/services/providers/modelengine_provider.py +++ b/backend/services/providers/modelengine_provider.py @@ -4,13 +4,21 @@ import aiohttp from consts.const import DEFAULT_LLM_MAX_TOKENS -from services.providers.base import AbstractModelProvider, _classify_provider_error +from services.providers.base import ( + AbstractModelProvider, + _classify_provider_error, + _extract_capacity_hints_from_raw, +) logger = logging.getLogger("model_provider") MODEL_ENGINE_NORTH_PREFIX = "open/router/v1" +def _extract_capacity_hints(raw: Dict) -> Dict: + return _extract_capacity_hints_from_raw(raw) + + def get_model_engine_raw_url(model_engine_url: str) -> str: """ Extract the raw base URL from a ModelEngine URL by stripping any API paths. @@ -96,14 +104,16 @@ async def get_models(self, provider_config: Dict) -> List[Dict]: continue if internal_type: - filtered_models.append({ + cleaned_model = { "id": model.get("id", ""), "model_type": internal_type, "model_tag": me_type, "max_tokens": DEFAULT_LLM_MAX_TOKENS if internal_type in ("llm", "vlm") else 0, "base_url": host, "api_key": api_key, - }) + } + cleaned_model.update(_extract_capacity_hints(model)) + filtered_models.append(cleaned_model) return filtered_models except Exception as e: diff --git a/backend/services/providers/silicon_provider.py b/backend/services/providers/silicon_provider.py index 1875b3949..e078f83a7 100644 --- a/backend/services/providers/silicon_provider.py +++ b/backend/services/providers/silicon_provider.py @@ -4,7 +4,11 @@ from consts.const import DEFAULT_LLM_MAX_TOKENS from consts.provider import SILICON_GET_URL -from services.providers.base import AbstractModelProvider, _classify_provider_error +from services.providers.base import ( + AbstractModelProvider, + _classify_provider_error, + _extract_capacity_hints_from_raw, +) SILICON_VLM_MODEL_KEYWORDS = ( @@ -33,6 +37,10 @@ SILICON_VLM_METADATA_KEYWORDS = ("image", "video", "vision", "visual") +def _extract_capacity_hints(raw: Dict) -> Dict: + return _extract_capacity_hints_from_raw(raw) + + def _contains_silicon_vlm_metadata(value) -> bool: if isinstance(value, str): lower_value = value.lower() @@ -107,6 +115,7 @@ async def get_models(self, provider_config: Dict) -> List[Dict]: # Annotate models with canonical fields expected downstream if provider_model_type in ("llm", "vlm"): for item in model_list: + item.update(_extract_capacity_hints(item)) item["model_tag"] = "chat" item["model_type"] = model_type item["max_tokens"] = DEFAULT_LLM_MAX_TOKENS diff --git a/backend/services/providers/tokenpony_provider.py b/backend/services/providers/tokenpony_provider.py index be2bb9c71..16adf0008 100644 --- a/backend/services/providers/tokenpony_provider.py +++ b/backend/services/providers/tokenpony_provider.py @@ -6,7 +6,11 @@ from consts.const import DEFAULT_LLM_MAX_TOKENS from consts.provider import TOKENPONY_GET_URL -from services.providers.base import AbstractModelProvider, _classify_provider_error +from services.providers.base import ( + AbstractModelProvider, + _classify_provider_error, + _extract_capacity_hints_from_raw, +) TOKENPONY_IMAGE_UNDERSTANDING_KEYWORDS = ( @@ -41,6 +45,10 @@ TOKENPONY_VIDEO_UNDERSTANDING_KEYWORDS = ("omni", "video") +def _extract_capacity_hints(raw: Dict) -> Dict: + return _extract_capacity_hints_from_raw(raw) + + def _has_keyword(text: str, keywords: tuple) -> bool: return any(keyword in text for keyword in keywords) @@ -126,6 +134,7 @@ async def get_models(self, provider_config: Dict) -> List[Dict]: "model_type": "", "max_tokens": DEFAULT_LLM_MAX_TOKENS } + cleaned_model.update(_extract_capacity_hints(model_obj)) # 1. rerank if 'rerank' in m_id: cleaned_model.update({"model_tag": "rerank", "model_type": "rerank"}) diff --git a/backend/services/tool_configuration_service.py b/backend/services/tool_configuration_service.py index 3cbf5edc5..0f5de35c3 100644 --- a/backend/services/tool_configuration_service.py +++ b/backend/services/tool_configuration_service.py @@ -415,8 +415,9 @@ async def get_tool_from_remote_mcp_server( input_schema["properties"][k]["type"] = "string" sanitized_tool_name = _sanitize_function_name(tool.name) + tool_description = tool.description or "" tool_info = ToolInfo(name=sanitized_tool_name, - description=tool.description, + description=tool_description, params=[], source=ToolSourceEnum.MCP.value, inputs=str(input_schema["properties"]), @@ -799,10 +800,12 @@ def _validate_local_tool( 'rerank_model': rerank_model, } tool_instance = tool_class(**params) - elif tool_name == "haotian_search": - # Haotian uses reranking_enable/reranking_model_name (not rerank/rerank_model_name) - # Must explicitly pass observer=None: if omitted, Python applies the FieldInfo default - # (not None), causing 'FieldInfo has no attr lang' errors in forward() + elif tool_name in ("haotian_search", "aidp_search"): + # Haotian and AIDP share the same instantiation shape: drop the + # backend-only rerank keys and explicitly set observer=None + # (otherwise Python falls back to the FieldInfo default, which + # later triggers "'FieldInfo' has no attribute 'lang'" in + # forward()). filtered_params = {k: v for k, v in instantiation_params.items() if k not in ["observer", "rerank_model", "rerank"]} filtered_params["observer"] = None @@ -812,7 +815,8 @@ def _validate_local_tool( raise ToolExecutionException( f"Tenant ID and User ID are required for {tool_name} validation") # get_vlm_model reads the first multimodal slot, now shown as image understanding. - image_to_text_model = get_vlm_model(tenant_id=tenant_id) + selected_model_id = instantiation_params.get("selected_model_id") + image_to_text_model = get_vlm_model(tenant_id=tenant_id, model_id=selected_model_id) vlm_display_name = getattr( image_to_text_model, 'display_name', None) set_monitoring_context(tenant_id=tenant_id) @@ -829,7 +833,8 @@ def _validate_local_tool( if not tenant_id or not user_id: raise ToolExecutionException( f"Tenant ID and User ID are required for {tool_name} validation") - video_understanding_model = get_video_understanding_model(tenant_id=tenant_id) + selected_model_id = instantiation_params.get("selected_model_id") + video_understanding_model = get_video_understanding_model(tenant_id=tenant_id, model_id=selected_model_id) model_display_name = getattr( video_understanding_model, 'display_name', None) set_monitoring_context(tenant_id=tenant_id) @@ -846,7 +851,8 @@ def _validate_local_tool( if not tenant_id or not user_id: raise ToolExecutionException( f"Tenant ID and User ID are required for {tool_name} validation") - long_text_to_text_model = get_llm_model(tenant_id=tenant_id) + selected_model_id = instantiation_params.get("selected_model_id") + long_text_to_text_model = get_llm_model(tenant_id=tenant_id, model_id=selected_model_id) llm_display_name = getattr( long_text_to_text_model, 'display_name', None) set_monitoring_context(tenant_id=tenant_id) diff --git a/backend/utils/auth_utils.py b/backend/utils/auth_utils.py index a7194f050..4ade6f211 100644 --- a/backend/utils/auth_utils.py +++ b/backend/utils/auth_utils.py @@ -6,8 +6,10 @@ from typing import Any, Dict, Optional, Tuple import jwt +import httpx from fastapi import Request from supabase import create_client +from supabase.lib.client_options import SyncClientOptions from consts.const import ( ASSET_OWNER_ROLE, @@ -249,10 +251,30 @@ def resolve_tenant_id_from_user_tenant_record(user_tenant: Dict[str, Any]) -> st return DEFAULT_TENANT_ID +def _build_supabase_options() -> SyncClientOptions: + """Build ClientOptions that bypass the system HTTP proxy. + + httpx 0.28 reads the Windows system proxy (e.g. Clash on 127.0.0.1:7897) + by default and routes every request through it. When the proxy cannot + reach a local service (such as GoTrue on http://localhost:8000) the + request hangs until the timeout, breaking login. + + Pass an explicit ``httpx.Client`` with ``trust_env=False`` and + ``proxy=None`` so Supabase always talks to ``SUPABASE_URL`` directly. + """ + http_client = httpx.Client( + trust_env=False, + proxy=None, + timeout=httpx.Timeout(30.0, connect=10.0), + follow_redirects=True, + ) + return SyncClientOptions(httpx_client=http_client) + + def get_supabase_client(): """Get Supabase client instance with regular key (user-context operations).""" try: - return create_client(SUPABASE_URL, SUPABASE_KEY) + return create_client(SUPABASE_URL, SUPABASE_KEY, options=_build_supabase_options()) except Exception as e: logging.error(f"Failed to create Supabase client: {str(e)}") return None @@ -261,7 +283,7 @@ def get_supabase_client(): def get_supabase_admin_client(): """Get Supabase client instance with service role key for admin operations.""" try: - return create_client(SUPABASE_URL, SERVICE_ROLE_KEY) + return create_client(SUPABASE_URL, SERVICE_ROLE_KEY, options=_build_supabase_options()) except Exception as e: logging.error(f"Failed to create Supabase admin client: {str(e)}") return None diff --git a/backend/utils/config_utils.py b/backend/utils/config_utils.py index 3fe6f3621..2d1c5572b 100644 --- a/backend/utils/config_utils.py +++ b/backend/utils/config_utils.py @@ -2,6 +2,7 @@ import logging from typing import Dict, Any +from pydantic import ValidationError from sqlalchemy.sql import func from database.model_management_db import get_model_by_model_id @@ -16,6 +17,9 @@ logger = logging.getLogger("config_utils") +CONTEXT_SOFT_LIMIT_RATIO_KEY = "context.soft_limit_ratio" + + def safe_value(value): """Helper function for processing configuration values""" if value is None: @@ -112,6 +116,39 @@ def get_app_config(self, key: str, default="", tenant_id: str | None = None): return tenant_config[key] return default + def get_capacity_reserve_policy(self, tenant_id: str | None = None): + """Resolve W2 reserve policy from tenant config. + + Missing `context.soft_limit_ratio` uses the code default. Invalid + configured values fail closed so production requests do not silently use + a different compaction envelope than operators configured. + """ + from nexent.core.models.capacity_budget import ( + CapacityReservePolicy, + InvalidReservePolicy, + ) + + if tenant_id is None: + logger.warning("No tenant_id specified when getting capacity reserve policy") + return CapacityReservePolicy() + + tenant_config = self.load_config(tenant_id) + raw_ratio = tenant_config.get(CONTEXT_SOFT_LIMIT_RATIO_KEY) + if raw_ratio in (None, ""): + return CapacityReservePolicy() + + try: + ratio = float(str(raw_ratio).strip()) + return CapacityReservePolicy( + soft_limit_ratio=ratio, + soft_limit_ratio_source="tenant_config", + ) + except (TypeError, ValueError, ValidationError) as exc: + raise InvalidReservePolicy( + f"{CONTEXT_SOFT_LIMIT_RATIO_KEY} must be a decimal in (0, 1], " + f"got {raw_ratio!r}" + ) from exc + def set_single_config(self, user_id: str | None = None, tenant_id: str | None = None, key: str | None = None, value: str | None = None, ): """Set configuration value in database with caching""" diff --git a/backend/utils/context_utils.py b/backend/utils/context_utils.py index 0c3af8915..4ddaa6d63 100644 --- a/backend/utils/context_utils.py +++ b/backend/utils/context_utils.py @@ -265,7 +265,6 @@ def _format_skills_description( def _format_tools_description( tools: Dict[str, Any], - knowledge_base_summary: Optional[str] = None, language: str = "zh", is_manager: bool = True, ) -> str: @@ -278,10 +277,16 @@ def _format_tools_description( """ if not tools: no_tools_msg = "- 当前没有可用的工具" if language == "zh" else "- No tools are currently available" - return no_tools_msg + prefix = "1. 工具\n" if language == "zh" else "1. Tools\n" + return prefix + no_tools_msg lines = [] + if language == "zh": + lines.append("1. 工具") + else: + lines.append("1. Tools") + if language == "zh": lines.append("- 你只能使用以下工具，不得使用任何其他工具：") else: @@ -319,15 +324,6 @@ def _format_tools_description( lines.append(f" Accepts input: {inputs}") lines.append(f" Returns output type: {output_type}") - # Knowledge base summary - if knowledge_base_summary: - if language == "zh": - lines.append("- knowledge_base_search工具只能使用以下知识库索引，请根据用户问题选择最相关的一个或多个知识库索引：") - lines.append(f" {knowledge_base_summary}") - else: - lines.append("- knowledge_base_search tool can only use the following knowledge base indexes, please select the most relevant one or more knowledge base indexes based on the user's question:") - lines.append(f" {knowledge_base_summary}") - # File URL usage guide lines.append("") if language == "zh": @@ -374,6 +370,11 @@ def _format_managed_agents_description( lines = [] + if language == "zh": + lines.append("2. 助手") + else: + lines.append("2. Agents") + if language == "zh": lines.append("你可以使用以下内部助手（通过函数调用方式协作）：") for name, agent in managed_agents.items(): @@ -461,6 +462,7 @@ def _format_external_agents_description( def _format_skills_usage_requirements( skills: List[Dict[str, str]], language: str = "zh", + is_manager: bool = True, ) -> str: """Format skills usage requirements section. @@ -469,10 +471,16 @@ def _format_skills_usage_requirements( """ if not skills: no_skills_msg = "- 当前没有可用的技能" if language == "zh" else "- No skills are currently available" - return no_skills_msg + prefix = "3. 技能\n" if language == "zh" else "3. Skills\n" + return prefix + no_skills_msg lines = [] + if language == "zh": + lines.append("3. 技能") + else: + lines.append("3. Skills") + if language == "zh": lines.append("- 你拥有上述 `` 中列出的技能。技能中引用的脚本通过 `run_skill_script()` 函数调用，该函数由平台提供，不需要导入。") lines.append("") @@ -533,7 +541,8 @@ def build_skeleton_header_component( """Build SystemPromptComponent for the header section. Section: "### 基本信息" / "### Basic Information" - Content: Agent identity, app name/description, user_id. + Content: Agent identity and app name/description. User identity is + request-scoped data and must not enter the managed stable prefix. Note: Current time is intentionally excluded from the system prompt so the static system prefix can hit the LLM KV/prompt cache across requests. The current time is injected on the user-message side instead (see CoreAgent.run). @@ -541,7 +550,7 @@ def build_skeleton_header_component( from nexent.core.agents.agent_model import SystemPromptComponent if language == "zh": - content = f"### 基本信息\n你是{app_name}，{app_description}，用户ID为{user_id}" + content = f"### 基本信息\n你是{app_name}，{app_description}" else: content = f"### Basic Information\nYou are {app_name}, {app_description}" @@ -555,17 +564,22 @@ def build_skeleton_header_component( def build_skeleton_duty_component( duty: str, language: str = "zh", + is_manager: bool = True, priority: int = 80, ) -> "SystemPromptComponent": """Build SystemPromptComponent for the duty section. Section: "### 核心职责" / "### Core Responsibilities" Content: Agent's primary duty + 5 safety principles + Note: Managed ZH agents use different safety principles than manager ZH agents. """ from nexent.core.agents.agent_model import SystemPromptComponent if language == "zh": - content = f"### 核心职责\n{duty}\n\n请注意，你应该遵守以下原则：\n行为安全：文件操作必须使用平台提供的专用工具，禁止使用代码直接修改工作空间中的文件；\n法律合规：遵守业务所在国家/地区的法律法规；\n政治中立：保持政治中立，不主动讨论政治话题；\n安全防护：不响应涉及武器制造、网络攻击、欺诈、恶意软件等危险行为的请求；\n伦理准则：拒绝仇恨言论、歧视性内容及违反社会公德和公认伦理标准的请求。" + if is_manager: + content = f"### 核心职责\n{duty}\n\n请注意，你应该遵守以下原则：\n行为安全：文件操作必须使用平台提供的专用工具，禁止使用代码直接修改工作空间中的文件；\n法律合规：遵守业务所在国家/地区的法律法规；\n政治中立：保持政治中立，不主动讨论政治话题；\n安全防护：不响应涉及武器制造、网络攻击、欺诈、恶意软件等危险行为的请求；\n伦理准则：拒绝仇恨言论、歧视性内容及违反社会公德和公认伦理标准的请求。" + else: + content = f"### 核心职责\n{duty}\n\n请注意，你应该遵守以下原则：\n行为安全：严禁直接执行代码进行文件的增删改操作，只能使用提供的文件操作类工具；\n法律合规：严格遵守服务地区的所有法律法规；\n政治中立：不讨论任何国家的政治体制、领导人评价或敏感历史事件；\n安全防护：不响应涉及武器制造、危险行为、隐私窃取等内容的请求；\n伦理准则：拒绝仇恨言论、歧视性内容及任何违反普世价值观的请求。" else: content = f"### Core Responsibilities\n{duty}\n\nPlease note that you should follow these principles:\nBehavioral Safety: File operations must use the platform-provided dedicated tools; direct code modification of workspace files is prohibited;\nLegal Compliance: Comply with laws and regulations of the business operating jurisdiction;\nPolitical Neutrality: Maintain political neutrality and avoid initiating political discussions;\nSecurity Protection: Do not respond to requests involving weapon manufacturing, cyberattacks, fraud, malware, or other dangerous activities;\nEthical Guidelines: Refuse hate speech, discriminatory content, and any requests that violate social morals and commonly accepted ethical standards." @@ -597,16 +611,23 @@ def build_skeleton_execution_flow_component( lines.append("要解决任务，你必须通过一系列步骤向前规划，以'思考：'和'代码：'序列循环进行。**注意：禁止在代码执行前输出'观察结果：'，观察结果只能由代码执行后产生。**") lines.append("") lines.append("1. 思考：") - lines.append(" - 分析当前任务状态和进展") - if is_manager and has_memory: + if is_manager: + lines.append(" - 分析当前任务状态和进展") + else: + lines.append(" - 确定需要使用哪些工具来获取信息或行动") + if has_memory: lines.append(" - 合理参考之前交互中的上下文记忆信息") - lines.append(" - 定下一步最佳行动（使用工具或分配给助手）") + if is_manager: + lines.append(" - 确定下一步最佳行动（使用工具或分配给助手）") lines.append(" - 解释你的决策逻辑和预期结果") lines.append("") lines.append("2. 代码：") lines.append(" - 用简单的Python编写代码") lines.append(" - 遵循python代码规范和python语法") - lines.append(" - 正确调用工具或助手解决问题") + if is_manager: + lines.append(" - 正确调用工具或助手解决问题") + else: + lines.append(" - 根据格式规范正确调用工具") lines.append(" - 考虑到代码执行与展示用户代码的区别，使用'代码'表达运行代码，使用'代码'表达展示代码") lines.append(" - 注意运行的代码不会被用户看到，所以如果用户需要看到代码，你需要使用'代码'表达展示代码。") lines.append(" - **重要**：代码执行后，系统会返回 \"Observation:\" 标记的内容（这是真实的执行结果）。请基于这些真实结果继续下一步思考，**不要在代码执行前自行编造观察结果**。") @@ -638,21 +659,31 @@ def build_skeleton_execution_flow_component( lines.append(" - 避免在Markdown中使用HTML标签，优先使用Markdown原生语法") lines.append(" - 代码块中的代码应保持原始格式，不要添加额外的转义字符") lines.append(" - 若未使用检索工具，则不添加任何引用标记") + if not is_manager: + lines.append("") + lines.append("注意最后生成的回答要语义连贯，信息清晰，可读性高。") else: lines = ["### Execution Process"] lines.append("To solve tasks, you must plan forward through a series of steps in a loop of 'Think:' and 'Code:' sequences. **IMPORTANT: You must NOT output 'Observe Results:' before code execution. Observation results can ONLY be generated after code execution.**") lines.append("") lines.append("1. Think:") - lines.append(" - Analyze current task status and progress") - if is_manager and has_memory: + if is_manager: + lines.append(" - Analyze current task status and progress") + else: + lines.append(" - Determine which tools need to be used to obtain information or take action") + if has_memory: lines.append(" - Reference relevant contextual memories from previous interactions when applicable") - lines.append(" - Determine the best next action (use tools or delegate to agents)") + if is_manager: + lines.append(" - Determine the best next action (use tools or delegate to agents)") lines.append(" - Explain your decision logic and expected results") lines.append("") lines.append("2. Code:") lines.append(" - Write code in simple Python") lines.append(" - Follow Python coding standards and Python syntax") - lines.append(" - Correctly call tools or agents to solve problems") + if is_manager: + lines.append(" - Correctly call tools or agents to solve problems") + else: + lines.append(" - Call tools correctly according to format specifications") lines.append(" - To distinguish between code execution and displaying user code, use 'code' for executing code and 'code' for displaying code") lines.append(" - Note that executed code is not visible to users. If users need to see the code, use 'code' for displaying code.") lines.append(" - **IMPORTANT**: After code execution, the system will return content with \"Observation:\" marker (this is the real execution result). Please continue your next thinking based on these real results. **Do NOT fabricate observation results before code execution.**") @@ -684,6 +715,9 @@ def build_skeleton_execution_flow_component( lines.append(" - Avoid using HTML tags in Markdown, prioritize native Markdown syntax") lines.append(" - Code in code blocks should maintain original format, do not add extra escape characters") lines.append(" - If no retrieval tools are used, do not add any reference marks") + if not is_manager: + lines.append("") + lines.append("Note that the final generated answer should be semantically coherent, with clear information and high readability.") content = "\n".join(lines) @@ -792,6 +826,35 @@ def build_skeleton_footer_component( ) +def build_available_resources_header_component( + is_manager: bool = True, + language: str = "zh", + priority: int = 55, +) -> "SystemPromptComponent": + """Build SystemPromptComponent for the Available Resources section header. + + Manager agents get a preamble restricting resources; managed agents get only the heading. + """ + from nexent.core.agents.agent_model import SystemPromptComponent + + if language == "zh": + if is_manager: + content = "### 可用资源\n你只能使用以下资源，不得使用任何其他工具或助手：" + else: + content = "### 可用资源" + else: + if is_manager: + content = "### Available Resources\nYou can only use the following resources, and may not use any other tools or agents:" + else: + content = "### Available Resources" + + return SystemPromptComponent( + content=content, + template_name="available_resources_header", + priority=priority, + ) + + # ============================================================================= # SECTION 3: Piecewise component builders (existing, enhanced) # ============================================================================= @@ -840,7 +903,6 @@ def build_tools_component( formatted_desc = _format_tools_description( tools, - knowledge_base_summary=knowledge_base_summary, language=language, is_manager=is_manager, ) @@ -923,6 +985,7 @@ def build_knowledge_base_component( knowledge_base_summary: str, kb_ids: Optional[List[str]] = None, priority: int = 10, + language: str = "zh", ) -> "KnowledgeBaseComponent": """Build KnowledgeBaseComponent from knowledge base summary. @@ -930,14 +993,24 @@ def build_knowledge_base_component( knowledge_base_summary: Summary text from knowledge bases kb_ids: List of knowledge base IDs used priority: Component priority for selection + language: Language code ('zh' or 'en') Returns: KnowledgeBaseComponent instance """ from nexent.core.agents.agent_model import KnowledgeBaseComponent + if knowledge_base_summary: + if language == "zh": + guidance = "knowledge_base_search 工具只能使用以下知识库索引，请根据用户的问题选择最相关的一个或多个知识库索引：\n" + else: + guidance = "knowledge_base_search tool can only use the following knowledge base indexes, please select the most relevant one or more knowledge base indexes based on the user's question:\n" + prefixed_summary = guidance + knowledge_base_summary + else: + prefixed_summary = knowledge_base_summary + return KnowledgeBaseComponent( - summary=knowledge_base_summary, + summary=prefixed_summary, kb_ids=kb_ids or [], priority=priority, ) @@ -1056,9 +1129,10 @@ def build_system_prompt_component( def build_skills_usage_component( skills: List[Dict[str, str]], language: str = "zh", + is_manager: bool = True, priority: int = 40, -) -> "SystemPromptComponent": - """Build SystemPromptComponent for skills usage requirements. +) -> "SkillsComponent": + """Build SkillsComponent for skills usage requirements. This is a skeleton-like component but its content depends on whether skills exist, so it's built dynamically. @@ -1066,17 +1140,18 @@ def build_skills_usage_component( Args: skills: List of skill dicts language: Language code ('zh' or 'en') + is_manager: Whether this is a manager agent priority: Component priority Returns: - SystemPromptComponent instance + SkillsComponent instance """ - from nexent.core.agents.agent_model import SystemPromptComponent + from nexent.core.agents.agent_model import SkillsComponent - content = _format_skills_usage_requirements(skills, language=language) - return SystemPromptComponent( - content=content, - template_name="skills_usage", + content = _format_skills_usage_requirements(skills, language=language, is_manager=is_manager) + return SkillsComponent( + skills=skills, + formatted_description=content, priority=priority, ) @@ -1150,20 +1225,22 @@ def build_context_components( Piecewise assembly: Each semantic section is emitted as a dedicated ContextComponent, assembled in the exact order matching Jinja2 templates. - Assembly order (12 sections): + Assembly order (15 sections): 1. Header (基本信息) 2. Memory (上下文记忆) - if memory_list exists 3. Duty (核心职责 + 安全准则) 4. Skills (可用技能 + 6步流程) - if skills exist 5. Execution Flow (执行流程 + 输出规范) - 6. Tools (可用资源/1. 工具 + 文件链接指南) - 7. Managed Agents (可用资源/2. 助手) - if managed_agents exist - 8. External Agents (外部助手) - if external_a2a_agents exist - 9. Agent Fallback (当前没有可用的助手) - if no agents - 10. Skills Usage (可用资源/3. 技能 + 使用要求) - 11. Constraint (资源使用要求) - 12. Code Norms (python代码规范) - 13. Footer (示例模板 + 结尾) + 6. Available Resources Header (可用资源 heading) + 7. Tools (可用资源/1. 工具 + 文件链接指南) + 8. Knowledge Base (知识库) - if knowledge_base_summary exists + 9. Managed Agents (可用资源/2. 助手) - if managed_agents exist + 10. External Agents (外部助手) - if external_a2a_agents exist + 11. Agent Fallback (当前没有可用的助手) - if no agents + 12. Skills Usage (可用资源/3. 技能 + 使用要求) + 13. Constraint (资源使用要求) + 14. Code Norms (python代码规范) + 15. Footer (示例模板 + 结尾) Note: The a330d815 short-circuit (if system_prompt: return [single]) has been REMOVED. All callers must provide raw params for piecewise assembly. @@ -1222,6 +1299,7 @@ def build_context_components( build_skeleton_duty_component( duty=duty, language=language, + is_manager=is_manager, ) ) @@ -1234,27 +1312,49 @@ def build_context_components( ) ) - # 5. Execution Flow + # 5. Execution Flow. Do not make stable instructions depend on whether a + # particular request happened to retrieve memory. components.append( build_skeleton_execution_flow_component( - memory_list=memory_list, + memory_list=None, language=language, is_manager=is_manager, ) ) - # 6. Tools + File URL Guide + # 6. Available Resources Header + components.append( + build_available_resources_header_component( + is_manager=is_manager, + language=language, + ) + ) + + # 7. Tools + File URL Guide if include_tools and tools: components.append( build_tools_component( tools=tools, - knowledge_base_summary=knowledge_base_summary, + # KB/RAG content is dynamic evidence and is emitted below as a + # user-role KnowledgeBaseComponent, not embedded in stable tool + # descriptions. + knowledge_base_summary=None, language=language, is_manager=is_manager, ) ) - # 7. Managed Agents (if exists) - manager only + # 8. Knowledge Base (if exists) + if include_knowledge_base and knowledge_base_summary: + components.append( + build_knowledge_base_component( + knowledge_base_summary=knowledge_base_summary, + kb_ids=kb_ids, + language=language, + ) + ) + + # 9. Managed Agents (if exists) - manager only if is_manager and include_managed_agents and managed_agents: components.append( build_managed_agents_component( @@ -1263,7 +1363,7 @@ def build_context_components( ) ) - # 8. External Agents (if exists) - manager only + # 10. External Agents (if exists) - manager only if is_manager and include_external_agents and external_a2a_agents: components.append( build_external_agents_component( @@ -1272,7 +1372,7 @@ def build_context_components( ) ) - # 9. Agent Fallback (if no agents available) - manager only + # 11. Agent Fallback (if no agents available) - manager only if is_manager and not managed_agents and not external_a2a_agents: fallback_comp = build_agent_fallback_component( managed_agents=managed_agents or {}, @@ -1282,16 +1382,17 @@ def build_context_components( if fallback_comp.content: # Only add if has content components.append(fallback_comp) - # 10. Skills Usage Requirements + # 12. Skills Usage Requirements if include_skills: components.append( build_skills_usage_component( skills=skills or [], language=language, + is_manager=is_manager, ) ) - # 11. Constraint + # 13. Constraint if constraint: components.append( build_skeleton_constraint_component( @@ -1300,7 +1401,7 @@ def build_context_components( ) ) - # 12. Code Norms + # 14. Code Norms components.append( build_skeleton_code_norms_component( language=language, @@ -1308,7 +1409,7 @@ def build_context_components( ) ) - # 13. Footer + # 15. Footer if few_shots: components.append( build_skeleton_footer_component( diff --git a/backend/utils/http_client_utils.py b/backend/utils/http_client_utils.py index 262c0a593..fd215c067 100644 --- a/backend/utils/http_client_utils.py +++ b/backend/utils/http_client_utils.py @@ -8,13 +8,15 @@ def create_httpx_client( headers: dict[str, str] | None = None, timeout: httpx.Timeout | None = None, auth: httpx.Auth | None = None, - **kwargs, + follow_redirects: bool = True, + **extra_kwargs, ) -> AsyncClient: return AsyncClient( headers=headers, timeout=timeout, auth=auth, + follow_redirects=follow_redirects, trust_env=False, verify=False, - **kwargs, + **extra_kwargs, ) diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 000000000..a5a013f2b --- /dev/null +++ b/deploy.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +usage() { + cat <<'USAGE' +Usage: + bash deploy.sh [--load-images] docker [docker deploy options] + bash deploy.sh [--load-images] k8s [k8s deploy options] + +This root entrypoint only forwards to the target-specific deploy script. +Implementation: deploy/deploy.sh + +Options: + --load-images Load Docker image tar files from ./images before deploying. + Defaults to off. +USAGE +} + +if [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ] || [ $# -eq 0 ]; then + usage + exit 0 +fi + +LOAD_IMAGES="false" +FORWARD_ARGS=() + +while [ $# -gt 0 ]; do + case "$1" in + --load-images) + LOAD_IMAGES="true" + shift + ;; + *) + FORWARD_ARGS+=("$1") + shift + ;; + esac +done + +if [ "${#FORWARD_ARGS[@]}" -eq 0 ]; then + usage + exit 0 +fi + +if [ "$LOAD_IMAGES" = "true" ]; then + LOAD_SCRIPT="$SCRIPT_DIR/load-images.sh" + if [ ! -f "$LOAD_SCRIPT" ]; then + echo "Error: --load-images requires $LOAD_SCRIPT" >&2 + exit 1 + fi + bash "$LOAD_SCRIPT" +fi + +exec bash "$SCRIPT_DIR/deploy/deploy.sh" "${FORWARD_ARGS[@]}" diff --git a/scripts/deployment/common.sh b/deploy/common/common.sh similarity index 88% rename from scripts/deployment/common.sh rename to deploy/common/common.sh index 006561553..db195f34a 100755 --- a/scripts/deployment/common.sh +++ b/deploy/common/common.sh @@ -5,7 +5,7 @@ # install environments. DEPLOYMENT_SCHEMA_VERSION="1" -DEPLOYMENT_COMPONENTS_DEFAULT="infrastructure,application" +DEPLOYMENT_COMPONENTS_DEFAULT="infrastructure,application,data-process,supabase" DEPLOYMENT_PORT_POLICY_DEFAULT="development" DEPLOYMENT_IMAGE_SOURCE_DEFAULT="general" DEPLOYMENT_REGISTRY_PROFILE_DEFAULT="general" @@ -27,6 +27,7 @@ DEPLOYMENT_LOADED_SCHEMA_VERSION="" DEPLOYMENT_LOADED_APP_VERSION="" DEPLOYMENT_CONFIG_FILE_LOADED="false" DEPLOYMENT_DOCKER_PORTS="" +DEPLOYMENT_ROOT_ENV="" deployment_component_list="infrastructure application data-process supabase terminal monitoring" deployment_port_policy_list="development production" @@ -69,6 +70,137 @@ deployment_trim() { printf '%s' "$value" } +deployment_validate_password() { + local password="$1" + + [ -n "$password" ] || return 1 + [ "${#password}" -ge 8 ] || return 1 + [[ "$password" =~ [A-Z] ]] || return 1 + [[ "$password" =~ [a-z] ]] || return 1 + [[ "$password" =~ [0-9] ]] || return 1 + return 0 +} + +deployment_password_validation_message() { + printf '%s\n' "Password must be at least 8 characters and include uppercase letters, lowercase letters, and numbers." +} + +deployment_ensure_root_env() { + local project_root="$1" + local docker_dir="${2:-$project_root/docker}" + local root_env="$project_root/.env" + local root_example="$project_root/.env.example" + local legacy_docker_env="$docker_dir/.env" + local legacy_docker_example="$docker_dir/.env.example" + + DEPLOYMENT_ROOT_ENV="$root_env" + export DEPLOYMENT_ROOT_ENV + + if [ -f "$root_env" ]; then + return 0 + fi + + if [ -f "$legacy_docker_env" ]; then + cp "$legacy_docker_env" "$root_env" + deployment_log "✅ Created root .env from legacy docker/.env" + return 0 + fi + + if [ -f "$root_example" ]; then + cp "$root_example" "$root_env" + deployment_log "✅ Created root .env from .env.example" + return 0 + fi + + if [ -f "$legacy_docker_example" ]; then + cp "$legacy_docker_example" "$root_env" + deployment_log "✅ Created root .env from legacy docker/.env.example" + return 0 + fi + + deployment_error ".env not found and no .env.example template is available" + return 1 +} + +deployment_source_root_env() { + local project_root="$1" + local docker_dir="${2:-$project_root/docker}" + + deployment_ensure_root_env "$project_root" "$docker_dir" || return 1 + + set -a + # shellcheck source=/dev/null + source "$DEPLOYMENT_ROOT_ENV" + set +a +} + +deployment_update_env_var_file() { + local env_file="$1" + local key="$2" + local value="$3" + local escaped_value + local current_value + + DEPLOYMENT_LAST_ENV_WRITE_CHANGED="false" + + touch "$env_file" + escaped_value=$(printf '%s' "$value" | sed -e 's/\\/\\\\/g' -e 's/&/\\&/g') + + if grep -q "^${key}=" "$env_file"; then + current_value="$(deployment_get_env_var_file "$env_file" "$key" || true)" + if [ "$current_value" = "$value" ]; then + return 0 + fi + sed -i.bak "s~^${key}=.*~${key}=\"${escaped_value}\"~" "$env_file" + rm -f "${env_file}.bak" + else + printf '%s="%s"\n' "$key" "$value" >> "$env_file" + fi + DEPLOYMENT_LAST_ENV_WRITE_CHANGED="true" +} + +deployment_get_env_var_file() { + local env_file="$1" + local key="$2" + local line value + + [ -f "$env_file" ] || return 1 + line="$(grep -E "^${key}=" "$env_file" | tail -n 1 || true)" + [ -n "$line" ] || return 1 + value="${line#*=}" + value="${value%$'\r'}" + value="$(printf '%s' "$value" | sed 's/[[:space:]]*$//')" + if [[ "$value" == \"*\" && "$value" == *\" ]]; then + value="${value#\"}" + value="${value%\"}" + elif [[ "$value" == \'*\' && "$value" == *\' ]]; then + value="${value#\'}" + value="${value%\'}" + fi + printf '%s' "$value" +} + +deployment_sha256_string() { + if command -v sha256sum >/dev/null 2>&1; then + printf '%s' "$1" | sha256sum | awk '{print $1}' + else + printf '%s' "$1" | shasum -a 256 | awk '{print $1}' + fi +} + +deployment_sha256_file() { + local file="$1" + [ -f "$file" ] || { + deployment_sha256_string "" + return 0 + } + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$file" | awk '{print $1}' + else + shasum -a 256 "$file" | awk '{print $1}' + fi +} + deployment_join_csv() { local sep="" local out="" @@ -102,10 +234,13 @@ deployment_init_defaults() { DEPLOYMENT_CONFIG_PATH="" DEPLOYMENT_USE_LOCAL_CONFIG="false" DEPLOYMENT_RECONFIGURE="false" + DEPLOYMENT_ROTATE_SECRETS="false" + DEPLOYMENT_REFRESH_ES_KEY="false" DEPLOYMENT_LOCAL_CONFIG_PATH="$(deployment_default_local_config_path)" DEPLOYMENT_LOADED_SCHEMA_VERSION="" DEPLOYMENT_LOADED_APP_VERSION="" DEPLOYMENT_CONFIG_FILE_LOADED="false" + DEPLOYMENT_CONFIG_VALUES_LOADED="false" DEPLOYMENT_DOCKER_PORTS="" unset DEPLOYMENT_COMPONENTS_EXPLICIT DEPLOYMENT_PORT_POLICY_EXPLICIT DEPLOYMENT_REGISTRY_PROFILE_EXPLICIT unset DEPLOYMENT_MONITORING_PROVIDER_EXPLICIT DEPLOYMENT_IMAGE_SOURCE_EXPLICIT DEPLOYMENT_APP_VERSION_EXPLICIT @@ -146,6 +281,14 @@ deployment_parse_common_args() { DEPLOYMENT_RECONFIGURE="true" shift ;; + --rotate-secrets) + DEPLOYMENT_ROTATE_SECRETS="true" + shift + ;; + --refresh-es-key) + DEPLOYMENT_REFRESH_ES_KEY="true" + shift + ;; --config) DEPLOYMENT_CONFIG_PATH="$2" shift 2 @@ -172,6 +315,7 @@ deployment_load_config_file() { local in_components="false" local components="" + local loaded_config_value="false" local line key value item while IFS= read -r line || [ -n "$line" ]; do line="${line%%#*}" @@ -197,57 +341,77 @@ deployment_load_config_file() { value="${value%\"}" value="${value#\"}" case "$key" in - portPolicy) DEPLOYMENT_PORT_POLICY="$value" ;; + portPolicy) + DEPLOYMENT_PORT_POLICY="$value" + loaded_config_value="true" + ;; schemaVersion) [ "$load_mode" = "apply" ] && DEPLOYMENT_LOADED_SCHEMA_VERSION="$value" + loaded_config_value="true" + ;; + imageSource) + DEPLOYMENT_IMAGE_SOURCE="$value" + loaded_config_value="true" + ;; + registryProfile) + DEPLOYMENT_REGISTRY_PROFILE="$value" + loaded_config_value="true" ;; - imageSource) DEPLOYMENT_IMAGE_SOURCE="$value" ;; - registryProfile) DEPLOYMENT_REGISTRY_PROFILE="$value" ;; appVersion) DEPLOYMENT_APP_VERSION="$value" [ "$load_mode" = "apply" ] && DEPLOYMENT_LOADED_APP_VERSION="$value" + loaded_config_value="true" + ;; + monitoringProvider) + DEPLOYMENT_MONITORING_PROVIDER="$value" + loaded_config_value="true" ;; - monitoringProvider) DEPLOYMENT_MONITORING_PROVIDER="$value" ;; esac fi done < "$config_file" - [ -n "$components" ] && DEPLOYMENT_COMPONENTS="$components" + if [ -n "$components" ]; then + DEPLOYMENT_COMPONENTS="$components" + loaded_config_value="true" + fi + [ "$loaded_config_value" = "true" ] && DEPLOYMENT_CONFIG_VALUES_LOADED="true" [ "$load_mode" = "apply" ] && DEPLOYMENT_CONFIG_FILE_LOADED="true" return 0 } deployment_apply_legacy_inputs() { - if [ -z "${DEPLOYMENT_COMPONENTS_EXPLICIT:-}" ]; then + if [ -z "${DEPLOYMENT_COMPONENTS_EXPLICIT:-}" ] && [ "$DEPLOYMENT_CONFIG_VALUES_LOADED" != "true" ]; then case "${DEPLOYMENT_VERSION:-}" in speed) deployment_warn "DEPLOYMENT_VERSION=speed is deprecated; use --components infrastructure,application." DEPLOYMENT_COMPONENTS="infrastructure,application" ;; full) - deployment_warn "DEPLOYMENT_VERSION=full is deprecated; use --components infrastructure,application,supabase." - DEPLOYMENT_COMPONENTS="infrastructure,application,supabase" + deployment_warn "DEPLOYMENT_VERSION=full is deprecated; use --components infrastructure,application,data-process,supabase." + DEPLOYMENT_COMPONENTS="infrastructure,application,data-process,supabase" ;; esac fi - case "${DEPLOYMENT_MODE:-}" in - development) - deployment_warn "DEPLOYMENT_MODE=development is deprecated; use --port-policy development." - [ -z "${DEPLOYMENT_PORT_POLICY_EXPLICIT:-}" ] && DEPLOYMENT_PORT_POLICY="development" - ;; - production) - deployment_warn "DEPLOYMENT_MODE=production is deprecated; use --port-policy production." - [ -z "${DEPLOYMENT_PORT_POLICY_EXPLICIT:-}" ] && DEPLOYMENT_PORT_POLICY="production" - ;; - infrastructure) - deployment_warn "DEPLOYMENT_MODE=infrastructure is deprecated; use --components infrastructure." - [ -z "${DEPLOYMENT_COMPONENTS_EXPLICIT:-}" ] && DEPLOYMENT_COMPONENTS="infrastructure" - [ -z "${DEPLOYMENT_PORT_POLICY_EXPLICIT:-}" ] && DEPLOYMENT_PORT_POLICY="development" - ;; - esac + if [ "$DEPLOYMENT_CONFIG_VALUES_LOADED" != "true" ]; then + case "${DEPLOYMENT_MODE:-}" in + development) + deployment_warn "DEPLOYMENT_MODE=development is deprecated; use --port-policy development." + [ -z "${DEPLOYMENT_PORT_POLICY_EXPLICIT:-}" ] && DEPLOYMENT_PORT_POLICY="development" + ;; + production) + deployment_warn "DEPLOYMENT_MODE=production is deprecated; use --port-policy production." + [ -z "${DEPLOYMENT_PORT_POLICY_EXPLICIT:-}" ] && DEPLOYMENT_PORT_POLICY="production" + ;; + infrastructure) + deployment_warn "DEPLOYMENT_MODE=infrastructure is deprecated; use --components infrastructure." + [ -z "${DEPLOYMENT_COMPONENTS_EXPLICIT:-}" ] && DEPLOYMENT_COMPONENTS="infrastructure" + [ -z "${DEPLOYMENT_PORT_POLICY_EXPLICIT:-}" ] && DEPLOYMENT_PORT_POLICY="development" + ;; + esac + fi - if [ -n "${IS_MAINLAND:-}" ] && [ -z "${DEPLOYMENT_REGISTRY_PROFILE_EXPLICIT:-}" ]; then + if [ -n "${IS_MAINLAND:-}" ] && [ -z "${DEPLOYMENT_REGISTRY_PROFILE_EXPLICIT:-}" ] && [ "$DEPLOYMENT_CONFIG_VALUES_LOADED" != "true" ]; then if [[ "$IS_MAINLAND" =~ ^[Yy]$ ]]; then deployment_warn "--is-mainland Y is deprecated; use --image-source mainland." DEPLOYMENT_IMAGE_SOURCE="mainland" @@ -1259,6 +1423,8 @@ deployment_prepare_config() { --registry-profile) DEPLOYMENT_REGISTRY_PROFILE_EXPLICIT="true" ;; --app-version|--version) DEPLOYMENT_APP_VERSION_EXPLICIT="true" ;; --monitoring-provider) DEPLOYMENT_MONITORING_PROVIDER_EXPLICIT="true" ;; + --rotate-secrets) DEPLOYMENT_ROTATE_SECRETS="true" ;; + --refresh-es-key) DEPLOYMENT_REFRESH_ES_KEY="true" ;; esac done diff --git a/scripts/deployment/config.example.yaml b/deploy/common/config.example.yaml similarity index 100% rename from scripts/deployment/config.example.yaml rename to deploy/common/config.example.yaml diff --git a/deploy/common/run-sql-migrations.sh b/deploy/common/run-sql-migrations.sh new file mode 100755 index 000000000..2a34b1a22 --- /dev/null +++ b/deploy/common/run-sql-migrations.sh @@ -0,0 +1,379 @@ +#!/usr/bin/env bash + +set -euo pipefail + +MIGRATION_DIR="${NEXENT_SQL_MIGRATION_DIR:-/opt/nexent/sql/migrations}" +INIT_SQL_FILE="${NEXENT_SQL_INIT_FILE:-/opt/nexent/sql/init.sql}" +MIGRATION_TABLE="${NEXENT_SQL_MIGRATION_TABLE:-nexent.schema_migrations}" +LOCK_KEY="${NEXENT_SQL_MIGRATION_LOCK_KEY:-nexent_sql_migrations}" +MANIFEST_SEPARATOR=$'\037' + +POSTGRES_HOST="${POSTGRES_HOST:-nexent-postgresql}" +POSTGRES_PORT="${POSTGRES_PORT:-5432}" +POSTGRES_USER="${POSTGRES_USER:-root}" +POSTGRES_DB="${POSTGRES_DB:-nexent}" +POSTGRES_PASSWORD="${NEXENT_POSTGRES_PASSWORD:-${POSTGRES_PASSWORD:-}}" + +MODE="${NEXENT_SQL_STARTUP_MODE:-migrate}" +case "${1:-}" in + --migrate) + MODE="migrate" + shift + ;; + --wait) + MODE="wait" + shift + ;; + --off) + MODE="off" + shift + ;; +esac + +log() { + printf '[sql-migrations] %s\n' "$*" +} + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | awk '{print $1}' + else + log "ERROR: sha256sum or shasum is required" + exit 1 + fi +} + +psql_base() { + PGPASSWORD="$POSTGRES_PASSWORD" psql \ + -h "$POSTGRES_HOST" \ + -p "$POSTGRES_PORT" \ + -U "$POSTGRES_USER" \ + -d "$POSTGRES_DB" \ + -v ON_ERROR_STOP=1 \ + "$@" +} + +escape_sql_literal() { + printf "%s" "$1" | sed "s/'/''/g" +} + +split_migration_table() { + MIGRATION_SCHEMA="${MIGRATION_TABLE%.*}" + MIGRATION_TABLE_NAME="${MIGRATION_TABLE##*.}" + if [ "$MIGRATION_SCHEMA" = "$MIGRATION_TABLE_NAME" ]; then + MIGRATION_SCHEMA="public" + fi + SQL_SEARCH_PATH="\"$MIGRATION_SCHEMA\", public" + if [ "$MIGRATION_SCHEMA" != "nexent" ]; then + SQL_SEARCH_PATH="\"nexent\", $SQL_SEARCH_PATH" + fi +} + +detect_app_version() { + if [ -n "${NEXENT_APP_VERSION:-}" ]; then + printf "%s" "$NEXENT_APP_VERSION" + elif [ -n "${APP_VERSION:-}" ]; then + printf "%s" "$APP_VERSION" + elif [ -f /opt/nexent/VERSION ]; then + sed -n '1p' /opt/nexent/VERSION + else + printf "" + fi +} + +wait_for_postgres() { + local timeout="${NEXENT_SQL_WAIT_TIMEOUT_SECONDS:-120}" + local start + start="$(date +%s)" + until psql_base -Atqc "SELECT 1" >/dev/null 2>&1; do + if [ $(( $(date +%s) - start )) -ge "$timeout" ]; then + log "ERROR: PostgreSQL did not become ready within ${timeout}s" + return 1 + fi + sleep 2 + done +} + +append_manifest_entry() { + local migration_id="$1" + local checksum="$2" + local source_file="$3" + printf '%s%s%s%s%s\n' "$migration_id" "$MANIFEST_SEPARATOR" "$checksum" "$MANIFEST_SEPARATOR" "$source_file" >> "$MIGRATION_MANIFEST_FILE" +} + +collect_one_migration() { + local file="$1" + local migration_id checksum + migration_id="$(basename "$file")" + checksum="$(sha256_file "$file")" + append_manifest_entry "$migration_id" "$checksum" "$file" +} + +collect_manifest() { + MIGRATION_MANIFEST_FILE="$(mktemp /tmp/nexent-sql-migration-manifest.XXXXXX)" + : > "$MIGRATION_MANIFEST_FILE" + + if [ -d "$MIGRATION_DIR" ]; then + local file + while IFS= read -r file; do + [ -n "$file" ] || continue + collect_one_migration "$file" + done < <(find -H "$MIGRATION_DIR" -maxdepth 1 -type f -name '*.sql' -print | sort -V) + else + log "migration directory not found: $MIGRATION_DIR" + fi +} + +append_migration_table_sql() { + cat >> "$MIGRATION_PLAN_FILE" <> "$MIGRATION_PLAN_FILE" <> "$MIGRATION_PLAN_FILE" <> "$MIGRATION_PLAN_FILE" < "$MIGRATION_PLAN_FILE" + append_migration_table_sql + cat >> "$MIGRATION_PLAN_FILE" <> "$MIGRATION_PLAN_FILE" + + psql_base -f "$MIGRATION_PLAN_FILE" + + if [ "$(manifest_count)" = "0" ]; then + log "no migration files found in $MIGRATION_DIR" + fi + log "migration check complete" +} + +cleanup() { + if [ -n "${MIGRATION_PLAN_FILE:-}" ]; then + rm -f "$MIGRATION_PLAN_FILE" + fi + if [ -n "${MIGRATION_MANIFEST_FILE:-}" ]; then + rm -f "$MIGRATION_MANIFEST_FILE" + fi +} + +main() { + case "$MODE" in + off) + log "SQL migration startup mode is off" + return 0 + ;; + migrate|wait) + ;; + *) + log "ERROR: unsupported NEXENT_SQL_STARTUP_MODE: $MODE" + return 1 + ;; + esac + + wait_for_postgres + split_migration_table + APP_VERSION_VALUE="$(detect_app_version)" + collect_manifest + trap cleanup EXIT + + case "$MODE" in + migrate) + run_migrate_mode + ;; + wait) + run_wait_mode + ;; + esac +} + +main "$@" diff --git a/deploy/common/start-backend.sh b/deploy/common/start-backend.sh new file mode 100755 index 000000000..a49d77661 --- /dev/null +++ b/deploy/common/start-backend.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SQL_STARTUP_MODE="${NEXENT_SQL_STARTUP_MODE:-off}" + +if [ -z "${NEXENT_SQL_STARTUP_MODE+x}" ] && [ -n "${NEXENT_RUN_SQL_MIGRATIONS:-}" ]; then + if [ "$NEXENT_RUN_SQL_MIGRATIONS" = "true" ]; then + SQL_STARTUP_MODE="migrate" + else + SQL_STARTUP_MODE="off" + fi +fi + +case "$SQL_STARTUP_MODE" in + migrate) + /opt/nexent/scripts/run-sql-migrations.sh --migrate + ;; + wait) + /opt/nexent/scripts/run-sql-migrations.sh --wait + ;; + off|"") + ;; + *) + printf '[start-backend] ERROR: unsupported NEXENT_SQL_STARTUP_MODE: %s\n' "$SQL_STARTUP_MODE" >&2 + exit 1 + ;; +esac + +exec "$@" diff --git a/deploy/common/version.sh b/deploy/common/version.sh new file mode 100755 index 000000000..1d12f404a --- /dev/null +++ b/deploy/common/version.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +deployment_project_root() { + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + cd "$script_dir/../.." && pwd +} + +deployment_read_version() { + local explicit="${1:-}" + if [ -n "$explicit" ]; then + printf '%s\n' "$explicit" + return 0 + fi + + local root version_file + root="$(deployment_project_root)" + version_file="$root/VERSION" + if [ -f "$version_file" ]; then + sed -n '1{s/[[:space:]]*$//;p;}' "$version_file" + return 0 + fi + + local const_file="$root/backend/consts/const.py" + if [ -f "$const_file" ]; then + local line + line="$(grep -E '^APP_VERSION[[:space:]]*=' "$const_file" | tail -n 1 || true)" + line="${line##*=}" + line="$(printf '%s' "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/^["'\'']//;s/["'\'']$//')" + [ -n "$line" ] && printf '%s\n' "$line" + return 0 + fi + + printf 'latest\n' +} diff --git a/deploy/deploy.sh b/deploy/deploy.sh new file mode 100755 index 000000000..6e4478984 --- /dev/null +++ b/deploy/deploy.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +usage() { + cat <<'USAGE' +Usage: + bash deploy.sh docker [docker deploy options] + bash deploy.sh k8s [k8s deploy options] + +Docker implementation: deploy/docker/deploy.sh +K8s implementation: deploy/k8s/deploy.sh +USAGE +} + +case "${1:-}" in + docker) + shift + exec bash "$SCRIPT_DIR/docker/deploy.sh" "$@" + ;; + k8s|kubernetes|helm) + shift + exec bash "$SCRIPT_DIR/k8s/deploy.sh" "$@" + ;; + --help|-h|"") + usage + ;; + *) + echo "Unknown deploy target: $1" >&2 + usage >&2 + exit 1 + ;; +esac diff --git a/docker/monitoring/grafana/dashboards/nexent-llm-agent.json b/deploy/docker/assets/monitoring/grafana/dashboards/nexent-llm-agent.json similarity index 100% rename from docker/monitoring/grafana/dashboards/nexent-llm-agent.json rename to deploy/docker/assets/monitoring/grafana/dashboards/nexent-llm-agent.json diff --git a/docker/monitoring/grafana/provisioning/dashboards/dashboards.yml b/deploy/docker/assets/monitoring/grafana/provisioning/dashboards/dashboards.yml similarity index 100% rename from docker/monitoring/grafana/provisioning/dashboards/dashboards.yml rename to deploy/docker/assets/monitoring/grafana/provisioning/dashboards/dashboards.yml diff --git a/docker/monitoring/grafana/provisioning/datasources/datasources.yml b/deploy/docker/assets/monitoring/grafana/provisioning/datasources/datasources.yml similarity index 100% rename from docker/monitoring/grafana/provisioning/datasources/datasources.yml rename to deploy/docker/assets/monitoring/grafana/provisioning/datasources/datasources.yml diff --git a/docker/monitoring/monitoring.env.example b/deploy/docker/assets/monitoring/monitoring.env.example similarity index 100% rename from docker/monitoring/monitoring.env.example rename to deploy/docker/assets/monitoring/monitoring.env.example diff --git a/docker/monitoring/otel-collector-config.yml b/deploy/docker/assets/monitoring/otel-collector-config.yml similarity index 100% rename from docker/monitoring/otel-collector-config.yml rename to deploy/docker/assets/monitoring/otel-collector-config.yml diff --git a/docker/monitoring/otel-collector-grafana-config.yml b/deploy/docker/assets/monitoring/otel-collector-grafana-config.yml similarity index 100% rename from docker/monitoring/otel-collector-grafana-config.yml rename to deploy/docker/assets/monitoring/otel-collector-grafana-config.yml diff --git a/docker/monitoring/otel-collector-langfuse-config.yml b/deploy/docker/assets/monitoring/otel-collector-langfuse-config.yml similarity index 100% rename from docker/monitoring/otel-collector-langfuse-config.yml rename to deploy/docker/assets/monitoring/otel-collector-langfuse-config.yml diff --git a/docker/monitoring/otel-collector-langsmith-config.yml b/deploy/docker/assets/monitoring/otel-collector-langsmith-config.yml similarity index 100% rename from docker/monitoring/otel-collector-langsmith-config.yml rename to deploy/docker/assets/monitoring/otel-collector-langsmith-config.yml diff --git a/docker/monitoring/otel-collector-phoenix-config.yml b/deploy/docker/assets/monitoring/otel-collector-phoenix-config.yml similarity index 100% rename from docker/monitoring/otel-collector-phoenix-config.yml rename to deploy/docker/assets/monitoring/otel-collector-phoenix-config.yml diff --git a/docker/monitoring/otel-collector-zipkin-config.yml b/deploy/docker/assets/monitoring/otel-collector-zipkin-config.yml similarity index 100% rename from docker/monitoring/otel-collector-zipkin-config.yml rename to deploy/docker/assets/monitoring/otel-collector-zipkin-config.yml diff --git a/docker/monitoring/tempo.yml b/deploy/docker/assets/monitoring/tempo.yml similarity index 100% rename from docker/monitoring/tempo.yml rename to deploy/docker/assets/monitoring/tempo.yml diff --git a/docker/official-skills-zip/analyze-image.zip b/deploy/docker/assets/official-skills-zip/analyze-image.zip similarity index 100% rename from docker/official-skills-zip/analyze-image.zip rename to deploy/docker/assets/official-skills-zip/analyze-image.zip diff --git a/docker/official-skills-zip/analyze-text-file.zip b/deploy/docker/assets/official-skills-zip/analyze-text-file.zip similarity index 100% rename from docker/official-skills-zip/analyze-text-file.zip rename to deploy/docker/assets/official-skills-zip/analyze-text-file.zip diff --git a/docker/official-skills-zip/create-docx.zip b/deploy/docker/assets/official-skills-zip/create-docx.zip similarity index 100% rename from docker/official-skills-zip/create-docx.zip rename to deploy/docker/assets/official-skills-zip/create-docx.zip diff --git a/docker/official-skills-zip/create-file-directory.zip b/deploy/docker/assets/official-skills-zip/create-file-directory.zip similarity index 100% rename from docker/official-skills-zip/create-file-directory.zip rename to deploy/docker/assets/official-skills-zip/create-file-directory.zip diff --git a/docker/official-skills-zip/delete-file-directory.zip b/deploy/docker/assets/official-skills-zip/delete-file-directory.zip similarity index 100% rename from docker/official-skills-zip/delete-file-directory.zip rename to deploy/docker/assets/official-skills-zip/delete-file-directory.zip diff --git a/docker/official-skills-zip/email-utils.zip b/deploy/docker/assets/official-skills-zip/email-utils.zip similarity index 100% rename from docker/official-skills-zip/email-utils.zip rename to deploy/docker/assets/official-skills-zip/email-utils.zip diff --git a/docker/official-skills-zip/list-directory.zip b/deploy/docker/assets/official-skills-zip/list-directory.zip similarity index 100% rename from docker/official-skills-zip/list-directory.zip rename to deploy/docker/assets/official-skills-zip/list-directory.zip diff --git a/docker/official-skills-zip/move-file-directory.zip b/deploy/docker/assets/official-skills-zip/move-file-directory.zip similarity index 100% rename from docker/official-skills-zip/move-file-directory.zip rename to deploy/docker/assets/official-skills-zip/move-file-directory.zip diff --git a/docker/official-skills-zip/read-file.zip b/deploy/docker/assets/official-skills-zip/read-file.zip similarity index 100% rename from docker/official-skills-zip/read-file.zip rename to deploy/docker/assets/official-skills-zip/read-file.zip diff --git a/docker/official-skills-zip/run-shell-ssh.zip b/deploy/docker/assets/official-skills-zip/run-shell-ssh.zip similarity index 100% rename from docker/official-skills-zip/run-shell-ssh.zip rename to deploy/docker/assets/official-skills-zip/run-shell-ssh.zip diff --git a/docker/official-skills-zip/search-datamate.zip b/deploy/docker/assets/official-skills-zip/search-datamate.zip similarity index 100% rename from docker/official-skills-zip/search-datamate.zip rename to deploy/docker/assets/official-skills-zip/search-datamate.zip diff --git a/docker/official-skills-zip/search-dify.zip b/deploy/docker/assets/official-skills-zip/search-dify.zip similarity index 100% rename from docker/official-skills-zip/search-dify.zip rename to deploy/docker/assets/official-skills-zip/search-dify.zip diff --git a/docker/official-skills-zip/search-idata.zip b/deploy/docker/assets/official-skills-zip/search-idata.zip similarity index 100% rename from docker/official-skills-zip/search-idata.zip rename to deploy/docker/assets/official-skills-zip/search-idata.zip diff --git a/docker/official-skills-zip/search-knowledge-base.zip b/deploy/docker/assets/official-skills-zip/search-knowledge-base.zip similarity index 100% rename from docker/official-skills-zip/search-knowledge-base.zip rename to deploy/docker/assets/official-skills-zip/search-knowledge-base.zip diff --git a/docker/official-skills-zip/search-web-exa.zip b/deploy/docker/assets/official-skills-zip/search-web-exa.zip similarity index 100% rename from docker/official-skills-zip/search-web-exa.zip rename to deploy/docker/assets/official-skills-zip/search-web-exa.zip diff --git a/docker/official-skills-zip/search-web-linkup.zip b/deploy/docker/assets/official-skills-zip/search-web-linkup.zip similarity index 100% rename from docker/official-skills-zip/search-web-linkup.zip rename to deploy/docker/assets/official-skills-zip/search-web-linkup.zip diff --git a/docker/official-skills-zip/search-web-tavily.zip b/deploy/docker/assets/official-skills-zip/search-web-tavily.zip similarity index 100% rename from docker/official-skills-zip/search-web-tavily.zip rename to deploy/docker/assets/official-skills-zip/search-web-tavily.zip diff --git a/docker/scripts/sync_skill_directory.py b/deploy/docker/assets/scripts/sync_skill_directory.py similarity index 95% rename from docker/scripts/sync_skill_directory.py rename to deploy/docker/assets/scripts/sync_skill_directory.py index d5819d251..26c62669b 100644 --- a/docker/scripts/sync_skill_directory.py +++ b/deploy/docker/assets/scripts/sync_skill_directory.py @@ -51,11 +51,20 @@ def get_env(key: str, default: str = "") -> str: def load_environment_from_host(): """ Load environment variables from host .env file. - Looks for .env in the same directory as this script's parent (docker/). + Looks for the project root .env first, with docker/.env as a legacy fallback. """ script_dir = Path(__file__).resolve().parent - docker_dir = script_dir.parent - env_file = docker_dir / ".env" + candidates = [] + explicit_env = os.environ.get("DEPLOYMENT_ROOT_ENV") + if explicit_env: + candidates.append(Path(explicit_env)) + candidates.extend([ + script_dir.parent.parent.parent.parent / ".env", # deploy/docker/assets/scripts + script_dir.parent.parent.parent / ".env", + script_dir.parent.parent / ".env", + script_dir.parent / ".env", + ]) + env_file = next((candidate for candidate in candidates if candidate.is_file()), candidates[0]) if env_file.is_file(): logger.info(f"Loading environment from: {env_file}") @@ -80,8 +89,17 @@ def get_root_dir() -> str: root_dir = get_env("ROOT_DIR") if not root_dir: script_dir = Path(__file__).resolve().parent - docker_dir = script_dir.parent - env_file = docker_dir / ".env" + candidates = [] + explicit_env = os.environ.get("DEPLOYMENT_ROOT_ENV") + if explicit_env: + candidates.append(Path(explicit_env)) + candidates.extend([ + script_dir.parent.parent.parent.parent / ".env", + script_dir.parent.parent.parent / ".env", + script_dir.parent.parent / ".env", + script_dir.parent / ".env", + ]) + env_file = next((candidate for candidate in candidates if candidate.is_file()), candidates[0]) if env_file.is_file(): with open(env_file, 'r') as f: for line in f: diff --git a/docker/scripts/sync_user_supabase2pg.py b/deploy/docker/assets/scripts/sync_user_supabase2pg.py similarity index 100% rename from docker/scripts/sync_user_supabase2pg.py rename to deploy/docker/assets/scripts/sync_user_supabase2pg.py diff --git a/docker/scripts/v180_sync_user_metadata.sh b/deploy/docker/assets/scripts/v180_sync_user_metadata.sh similarity index 100% rename from docker/scripts/v180_sync_user_metadata.sh rename to deploy/docker/assets/scripts/v180_sync_user_metadata.sh diff --git a/docker/scripts/v220_sync_skill_directory.sh b/deploy/docker/assets/scripts/v220_sync_skill_directory.sh similarity index 76% rename from docker/scripts/v220_sync_skill_directory.sh rename to deploy/docker/assets/scripts/v220_sync_skill_directory.sh index 572ffeb30..802790d9c 100644 --- a/docker/scripts/v220_sync_skill_directory.sh +++ b/deploy/docker/assets/scripts/v220_sync_skill_directory.sh @@ -56,9 +56,18 @@ if [ ! -f "$SCRIPT_PATH" ]; then exit 1 fi -# Load environment from .env if exists -ENV_FILE="${SCRIPT_DIR}/../.env" -if [ -f "$ENV_FILE" ]; then +# Load environment from project root .env if exists. The script may run from +# deploy/docker/assets/scripts or from the copied ROOT_DIR/scripts directory. +ENV_FILE="${DEPLOYMENT_ROOT_ENV:-}" +if [ -z "$ENV_FILE" ]; then + for candidate in "${SCRIPT_DIR}/../../../../.env" "${SCRIPT_DIR}/../../../.env" "${SCRIPT_DIR}/../../.env"; do + if [ -f "$candidate" ]; then + ENV_FILE="$candidate" + break + fi + done +fi +if [ -n "$ENV_FILE" ] && [ -f "$ENV_FILE" ]; then log_info "Loading environment from: $ENV_FILE" set -a source "$ENV_FILE" diff --git a/docker/volumes/api/kong.yml b/deploy/docker/assets/volumes/api/kong.yml similarity index 100% rename from docker/volumes/api/kong.yml rename to deploy/docker/assets/volumes/api/kong.yml diff --git a/docker/volumes/functions/hello/index.ts b/deploy/docker/assets/volumes/functions/hello/index.ts similarity index 100% rename from docker/volumes/functions/hello/index.ts rename to deploy/docker/assets/volumes/functions/hello/index.ts diff --git a/docker/volumes/functions/main/index.ts b/deploy/docker/assets/volumes/functions/main/index.ts similarity index 100% rename from docker/volumes/functions/main/index.ts rename to deploy/docker/assets/volumes/functions/main/index.ts diff --git a/docker/volumes/pooler/pooler.exs b/deploy/docker/assets/volumes/pooler/pooler.exs similarity index 100% rename from docker/volumes/pooler/pooler.exs rename to deploy/docker/assets/volumes/pooler/pooler.exs diff --git a/docker/docker-compose-monitoring.yml b/deploy/docker/compose/docker-compose-monitoring.yml similarity index 96% rename from docker/docker-compose-monitoring.yml rename to deploy/docker/compose/docker-compose-monitoring.yml index 976a57c97..cd6805a2a 100644 --- a/docker/docker-compose-monitoring.yml +++ b/deploy/docker/compose/docker-compose-monitoring.yml @@ -11,7 +11,7 @@ services: LANGSMITH_PROJECT: ${LANGSMITH_PROJECT:-nexent} LANGSMITH_OTLP_TRACES_ENDPOINT: ${LANGSMITH_OTLP_TRACES_ENDPOINT:-https://api.smith.langchain.com/otel/v1/traces} volumes: - - ${OTEL_COLLECTOR_CONFIG_FILE:-./monitoring/otel-collector-config.yml}:/etc/otel-collector-config.yml + - ${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-config.yml}:/etc/otel-collector-config.yml ports: - "${OTEL_COLLECTOR_GRPC_PORT:-4317}:4317" - "${OTEL_COLLECTOR_HTTP_PORT:-4318}:4318" @@ -40,7 +40,7 @@ services: profiles: ["grafana"] command: ["--config.file=/etc/tempo.yml"] volumes: - - ./monitoring/tempo.yml:/etc/tempo.yml:ro + - ../assets/monitoring/tempo.yml:/etc/tempo.yml:ro - tempo-data:/var/tempo ports: - "${TEMPO_PORT:-3200}:3200" @@ -60,8 +60,8 @@ services: GF_PLUGINS_PREINSTALL_AUTO_UPDATE: "false" volumes: - grafana-data:/var/lib/grafana - - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro - - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + - ../assets/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ../assets/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro ports: - "${GRAFANA_PORT:-3002}:3000" depends_on: diff --git a/docker/docker-compose-supabase.prod.yml b/deploy/docker/compose/docker-compose-supabase.prod.yml similarity index 83% rename from docker/docker-compose-supabase.prod.yml rename to deploy/docker/compose/docker-compose-supabase.prod.yml index 6ad7ac134..daec58ad4 100644 --- a/docker/docker-compose-supabase.prod.yml +++ b/deploy/docker/compose/docker-compose-supabase.prod.yml @@ -6,7 +6,9 @@ services: volumes: - $ROOT_DIR/volumes/api/kong.yml:/home/kong/temp.yml networks: - - nexent + nexent: + aliases: + - nexent-supabase-kong depends_on: db: condition: service_healthy @@ -90,16 +92,20 @@ services: image: ${SUPABASE_DB} restart: unless-stopped volumes: - - $ROOT_DIR/volumes/db/realtime.sql:/docker-entrypoint-initdb.d/migrations/99-realtime.sql - - $ROOT_DIR/volumes/db/webhooks.sql:/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql - - $ROOT_DIR/volumes/db/roles.sql:/docker-entrypoint-initdb.d/init-scripts/99-roles.sql - - $ROOT_DIR/volumes/db/jwt.sql:/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql + - ../../sql/supabase/realtime.sql:/docker-entrypoint-initdb.d/migrations/99-realtime.sql:ro + - ../../sql/supabase/logs.sql:/docker-entrypoint-initdb.d/migrations/99-logs.sql:ro + - ../../sql/supabase/pooler.sql:/docker-entrypoint-initdb.d/migrations/99-pooler.sql:ro + - ../../sql/supabase/webhooks.sql:/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql:ro + - ../../sql/supabase/roles.sql:/docker-entrypoint-initdb.d/init-scripts/99-roles.sql:ro + - ../../sql/supabase/jwt.sql:/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql:ro - $ROOT_DIR/volumes/db/data:/var/lib/postgresql/data - - $ROOT_DIR/volumes/db/_supabase.sql:/docker-entrypoint-initdb.d/migrations/97-_supabase.sql + - ../../sql/supabase/_supabase.sql:/docker-entrypoint-initdb.d/migrations/97-_supabase.sql:ro - $ROOT_DIR/volumes/logs:/var/log/postgresql - db-config:/etc/postgresql-custom networks: - - nexent + nexent: + aliases: + - nexent-supabase-db healthcheck: test: [ diff --git a/docker/docker-compose-supabase.yml b/deploy/docker/compose/docker-compose-supabase.yml similarity index 84% rename from docker/docker-compose-supabase.yml rename to deploy/docker/compose/docker-compose-supabase.yml index b781b4444..61a326bea 100644 --- a/docker/docker-compose-supabase.yml +++ b/deploy/docker/compose/docker-compose-supabase.yml @@ -9,7 +9,9 @@ services: volumes: - $ROOT_DIR/volumes/api/kong.yml:/home/kong/temp.yml networks: - - nexent + nexent: + aliases: + - nexent-supabase-kong depends_on: db: condition: service_healthy @@ -95,16 +97,20 @@ services: ports: - ${SUPABASE_POSTGRES_PORT}:${SUPABASE_POSTGRES_PORT} volumes: - - $ROOT_DIR/volumes/db/realtime.sql:/docker-entrypoint-initdb.d/migrations/99-realtime.sql - - $ROOT_DIR/volumes/db/webhooks.sql:/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql - - $ROOT_DIR/volumes/db/roles.sql:/docker-entrypoint-initdb.d/init-scripts/99-roles.sql - - $ROOT_DIR/volumes/db/jwt.sql:/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql + - ../../sql/supabase/realtime.sql:/docker-entrypoint-initdb.d/migrations/99-realtime.sql:ro + - ../../sql/supabase/logs.sql:/docker-entrypoint-initdb.d/migrations/99-logs.sql:ro + - ../../sql/supabase/pooler.sql:/docker-entrypoint-initdb.d/migrations/99-pooler.sql:ro + - ../../sql/supabase/webhooks.sql:/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql:ro + - ../../sql/supabase/roles.sql:/docker-entrypoint-initdb.d/init-scripts/99-roles.sql:ro + - ../../sql/supabase/jwt.sql:/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql:ro - $ROOT_DIR/volumes/db/data:/var/lib/postgresql/data - - $ROOT_DIR/volumes/db/_supabase.sql:/docker-entrypoint-initdb.d/migrations/97-_supabase.sql + - ../../sql/supabase/_supabase.sql:/docker-entrypoint-initdb.d/migrations/97-_supabase.sql:ro - $ROOT_DIR/volumes/logs:/var/log/postgresql - db-config:/etc/postgresql-custom networks: - - nexent + nexent: + aliases: + - nexent-supabase-db healthcheck: test: [ diff --git a/docker/docker-compose.dev.yml b/deploy/docker/compose/docker-compose.dev.yml similarity index 92% rename from docker/docker-compose.dev.yml rename to deploy/docker/compose/docker-compose.dev.yml index f23e4210c..a0ed009a8 100644 --- a/docker/docker-compose.dev.yml +++ b/deploy/docker/compose/docker-compose.dev.yml @@ -9,7 +9,7 @@ services: # - "5010:5010" # - "5013:5013" # volumes: -# - ../:/opt/ +# - ../../../:/opt/ # - /opt/backend/.venv/ # - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent # environment: @@ -43,7 +43,7 @@ services: ports: - "5012:5012" volumes: - - ../:/opt/:cached + - ../../../:/opt/:cached - /opt/backend/.venv/ - ${ROOT_DIR}:/mnt/nexent-data environment: @@ -51,7 +51,7 @@ services: PATH: "/usr/local/bin:/usr/bin/:/opt/backend/.venv/bin:${PATH}" VIRTUAL_ENV: "/opt/backend/.venv" env_file: - - .env + - ../../../.env networks: - nexent user: root @@ -79,8 +79,8 @@ services: # ports: # - "3000:3000" # volumes: -# - ../frontend:/opt/frontend:cached -# - ../frontend/node_modules:/opt/frontend/node_modules:cached +# - ../../../frontend:/opt/frontend:cached +# - ../../../frontend/node_modules:/opt/frontend/node_modules:cached # environment: # - HTTP_BACKEND=http://nexent:5010 # - WS_BACKEND=ws://nexent:5010 diff --git a/docker/docker-compose.prod.yml b/deploy/docker/compose/docker-compose.prod.yml similarity index 85% rename from docker/docker-compose.prod.yml rename to deploy/docker/compose/docker-compose.prod.yml index 29bd41d9f..2ee277db6 100644 --- a/docker/docker-compose.prod.yml +++ b/deploy/docker/compose/docker-compose.prod.yml @@ -57,9 +57,7 @@ services: POSTGRES_DB: ${POSTGRES_DB} volumes: - ${ROOT_DIR}/postgresql/data:/var/lib/postgresql/data - - ./init.sql:/docker-entrypoint-initdb.d/init.sql - security_opt: - - seccomp:unconfined + - ../../sql/init.sql:/docker-entrypoint-initdb.d/init.sql:ro restart: always logging: driver: "json-file" @@ -75,16 +73,19 @@ services: restart: always volumes: - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent + - ../../sql:/opt/nexent/sql:ro - ${ROOT_DIR}/skills:/mnt/nexent-data/skills - ${ROOT_DIR}/openssh-server/ssh-keys:/opt/ssh-keys:ro - ${ROOT_DIR}/scripts/sync_user_supabase2pg.py:/opt/sync_user_supabase2pg.py:ro - /var/run/docker.sock:/var/run/docker.sock:ro # Docker socket for MCP container management environment: <<: [*minio-vars, *es-vars] + NEXENT_SQL_STARTUP_MODE: migrate + NEXENT_SQL_FILES_CHECKSUM: ${NEXENT_SQL_FILES_CHECKSUM:-} skip_proxy: "true" UMASK: 0022 env_file: - - .env + - ../../../.env user: root depends_on: nexent-elasticsearch: @@ -96,7 +97,7 @@ services: max-file: "3" # Maximum number of log files to keep networks: - nexent - entrypoint: ["/bin/bash", "-c", "python backend/config_service.py"] + entrypoint: ["/opt/nexent/scripts/start-backend.sh", "python", "backend/config_service.py"] nexent-runtime: image: ${NEXENT_IMAGE} @@ -104,14 +105,17 @@ services: restart: always volumes: - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent + - ../../sql:/opt/nexent/sql:ro - ${ROOT_DIR}/skills:/mnt/nexent-data/skills - ${ROOT_DIR}/openssh-server/ssh-keys:/opt/ssh-keys:ro environment: <<: [*minio-vars, *es-vars] + NEXENT_SQL_STARTUP_MODE: wait + NEXENT_SQL_FILES_CHECKSUM: ${NEXENT_SQL_FILES_CHECKSUM:-} skip_proxy: "true" UMASK: 0022 env_file: - - .env + - ../../../.env user: root depends_on: nexent-elasticsearch: @@ -123,7 +127,7 @@ services: max-file: "3" # Maximum number of log files to keep networks: - nexent - entrypoint: ["/bin/bash", "-c", "python backend/runtime_service.py"] + entrypoint: ["/opt/nexent/scripts/start-backend.sh", "python", "backend/runtime_service.py"] nexent-mcp: image: ${NEXENT_IMAGE} @@ -131,13 +135,16 @@ services: restart: always volumes: - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent + - ../../sql:/opt/nexent/sql:ro - ${ROOT_DIR}/openssh-server/ssh-keys:/opt/ssh-keys:ro environment: <<: [*minio-vars, *es-vars] + NEXENT_SQL_STARTUP_MODE: wait + NEXENT_SQL_FILES_CHECKSUM: ${NEXENT_SQL_FILES_CHECKSUM:-} skip_proxy: "true" UMASK: 0022 env_file: - - .env + - ../../../.env user: root depends_on: nexent-elasticsearch: @@ -149,7 +156,7 @@ services: max-file: "3" # Maximum number of log files to keep networks: - nexent - entrypoint: ["/bin/bash", "-c", "python backend/mcp_service.py"] + entrypoint: ["/opt/nexent/scripts/start-backend.sh", "python", "backend/mcp_service.py"] nexent-northbound: image: ${NEXENT_IMAGE} @@ -157,14 +164,17 @@ services: restart: always volumes: - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent + - ../../sql:/opt/nexent/sql:ro - ${ROOT_DIR}/skills:/mnt/nexent-data/skills - ${ROOT_DIR}/openssh-server/ssh-keys:/opt/ssh-keys:ro environment: <<: [*minio-vars, *es-vars] + NEXENT_SQL_STARTUP_MODE: wait + NEXENT_SQL_FILES_CHECKSUM: ${NEXENT_SQL_FILES_CHECKSUM:-} skip_proxy: "true" UMASK: 0022 env_file: - - .env + - ../../../.env user: root depends_on: nexent-elasticsearch: @@ -178,7 +188,7 @@ services: - nexent ports: - "5013:5013" # Northbound API port exposed for external A2A access - entrypoint: ["/bin/bash", "-c", "python backend/northbound_service.py"] + entrypoint: ["/opt/nexent/scripts/start-backend.sh", "python", "backend/northbound_service.py"] nexent-web: image: ${NEXENT_WEB_IMAGE} @@ -203,20 +213,22 @@ services: nexent-data-process: image: ${NEXENT_DATA_PROCESS_IMAGE} container_name: nexent-data-process - command: bash restart: always privileged: true volumes: - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent + - ../../sql:/opt/nexent/sql:ro environment: <<: [*proxy-vars, *es-vars, *minio-vars] + NEXENT_SQL_STARTUP_MODE: off + NEXENT_SQL_FILES_CHECKSUM: ${NEXENT_SQL_FILES_CHECKSUM:-} DOCKER_ENVIRONMENT: "true" DISABLE_RAY_DASHBOARD: ${DISABLE_RAY_DASHBOARD:-false} DISABLE_CELERY_FLOWER: ${DISABLE_CELERY_FLOWER:-false} PYTHONPATH: "/opt/backend" skip_proxy: "true" env_file: - - .env + - ../../../.env depends_on: redis: condition: service_healthy @@ -231,7 +243,7 @@ services: - nexent entrypoint: > /bin/sh -c " - python /opt/backend/data_process_service.py || (cd /opt/backend && OPENBLAS_NUM_THREADS=1 UVICORN_LOOP=asyncio uvicorn data_process_service:app --host 0.0.0.0 --port 5012) + /opt/nexent/scripts/start-backend.sh /bin/sh -c 'python /opt/backend/data_process_service.py || (cd /opt/backend && OPENBLAS_NUM_THREADS=1 UVICORN_LOOP=asyncio uvicorn data_process_service:app --host 0.0.0.0 --port 5012)' " redis: diff --git a/docker/docker-compose.yml b/deploy/docker/compose/docker-compose.yml similarity index 86% rename from docker/docker-compose.yml rename to deploy/docker/compose/docker-compose.yml index fd3851ab4..f7afe78ad 100644 --- a/docker/docker-compose.yml +++ b/deploy/docker/compose/docker-compose.yml @@ -64,7 +64,7 @@ services: POSTGRES_DB: ${POSTGRES_DB} volumes: - ${ROOT_DIR}/postgresql/data:/var/lib/postgresql/data - - ./init.sql:/docker-entrypoint-initdb.d/init.sql + - ../../sql/init.sql:/docker-entrypoint-initdb.d/init.sql:ro ports: - "5434:5432" security_opt: @@ -86,16 +86,19 @@ services: - "5010:5010" # Config service port volumes: - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent + - ../../sql:/opt/nexent/sql:ro - ${ROOT_DIR}/skills:/mnt/nexent-data/skills - ${ROOT_DIR}/openssh-server/ssh-keys:/opt/ssh-keys:ro - ${ROOT_DIR}/scripts/sync_user_supabase2pg.py:/opt/sync_user_supabase2pg.py:ro - /var/run/docker.sock:/var/run/docker.sock:ro # Docker socket for MCP container management environment: <<: [*minio-vars, *es-vars] + NEXENT_SQL_STARTUP_MODE: migrate + NEXENT_SQL_FILES_CHECKSUM: ${NEXENT_SQL_FILES_CHECKSUM:-} skip_proxy: "true" UMASK: 0022 env_file: - - .env + - ../../../.env user: root depends_on: nexent-elasticsearch: @@ -107,7 +110,7 @@ services: max-file: "3" # Maximum number of log files to keep networks: - nexent - entrypoint: ["/bin/bash", "-c", "python backend/config_service.py"] + entrypoint: ["/opt/nexent/scripts/start-backend.sh", "python", "backend/config_service.py"] nexent-runtime: image: ${NEXENT_IMAGE} @@ -117,14 +120,17 @@ services: - "5014:5014" # Runtime service port volumes: - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent + - ../../sql:/opt/nexent/sql:ro - ${ROOT_DIR}/skills:/mnt/nexent-data/skills - ${ROOT_DIR}/openssh-server/ssh-keys:/opt/ssh-keys:ro environment: <<: [*minio-vars, *es-vars] + NEXENT_SQL_STARTUP_MODE: wait + NEXENT_SQL_FILES_CHECKSUM: ${NEXENT_SQL_FILES_CHECKSUM:-} skip_proxy: "true" UMASK: 0022 env_file: - - .env + - ../../../.env user: root depends_on: nexent-elasticsearch: @@ -136,7 +142,7 @@ services: max-file: "3" # Maximum number of log files to keep networks: - nexent - entrypoint: ["/bin/bash", "-c", "python backend/runtime_service.py"] + entrypoint: ["/opt/nexent/scripts/start-backend.sh", "python", "backend/runtime_service.py"] nexent-mcp: image: ${NEXENT_IMAGE} @@ -147,13 +153,16 @@ services: - "5015:5015" # MCP management API port volumes: - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent + - ../../sql:/opt/nexent/sql:ro - ${ROOT_DIR}/openssh-server/ssh-keys:/opt/ssh-keys:ro environment: <<: [*minio-vars, *es-vars] + NEXENT_SQL_STARTUP_MODE: wait + NEXENT_SQL_FILES_CHECKSUM: ${NEXENT_SQL_FILES_CHECKSUM:-} skip_proxy: "true" UMASK: 0022 env_file: - - .env + - ../../../.env user: root depends_on: nexent-elasticsearch: @@ -165,7 +174,7 @@ services: max-file: "3" # Maximum number of log files to keep networks: - nexent - entrypoint: ["/bin/bash", "-c", "python backend/mcp_service.py"] + entrypoint: ["/opt/nexent/scripts/start-backend.sh", "python", "backend/mcp_service.py"] nexent-northbound: image: ${NEXENT_IMAGE} @@ -175,14 +184,17 @@ services: - "5013:5013" # Northbound service port volumes: - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent + - ../../sql:/opt/nexent/sql:ro - ${ROOT_DIR}/skills:/mnt/nexent-data/skills - ${ROOT_DIR}/openssh-server/ssh-keys:/opt/ssh-keys:ro environment: <<: [*minio-vars, *es-vars] + NEXENT_SQL_STARTUP_MODE: wait + NEXENT_SQL_FILES_CHECKSUM: ${NEXENT_SQL_FILES_CHECKSUM:-} skip_proxy: "true" UMASK: 0022 env_file: - - .env + - ../../../.env user: root depends_on: nexent-elasticsearch: @@ -194,7 +206,7 @@ services: max-file: "3" # Maximum number of log files to keep networks: - nexent - entrypoint: ["/bin/bash", "-c", "python backend/northbound_service.py"] + entrypoint: ["/opt/nexent/scripts/start-backend.sh", "python", "backend/northbound_service.py"] nexent-web: image: ${NEXENT_WEB_IMAGE} @@ -220,7 +232,6 @@ services: nexent-data-process: image: ${NEXENT_DATA_PROCESS_IMAGE} container_name: nexent-data-process - command: bash restart: always privileged: true ports: @@ -229,13 +240,16 @@ services: - "8265:8265" # Ray Dashboardport volumes: - ${NEXENT_USER_DIR:-$HOME/nexent}:/mnt/nexent + - ../../sql:/opt/nexent/sql:ro environment: <<: [*proxy-vars, *es-vars, *minio-vars] + NEXENT_SQL_STARTUP_MODE: off + NEXENT_SQL_FILES_CHECKSUM: ${NEXENT_SQL_FILES_CHECKSUM:-} DOCKER_ENVIRONMENT: "true" PYTHONPATH: "/opt/backend" skip_proxy: "true" env_file: - - .env + - ../../../.env depends_on: redis: condition: service_healthy @@ -245,7 +259,7 @@ services: - nexent entrypoint: > /bin/sh -c " - python /opt/backend/data_process_service.py || (cd /opt/backend && OPENBLAS_NUM_THREADS=1 UVICORN_LOOP=asyncio uvicorn data_process_service:app --host 0.0.0.0 --port 5012) + /opt/nexent/scripts/start-backend.sh /bin/sh -c 'python /opt/backend/data_process_service.py || (cd /opt/backend && OPENBLAS_NUM_THREADS=1 UVICORN_LOOP=asyncio uvicorn data_process_service:app --host 0.0.0.0 --port 5012)' " logging: diff --git a/docker/create-su.sh b/deploy/docker/create-su.sh similarity index 97% rename from docker/create-su.sh rename to deploy/docker/create-su.sh index 639e64553..506570f42 100755 --- a/docker/create-su.sh +++ b/deploy/docker/create-su.sh @@ -7,11 +7,13 @@ # and return appropriate exit codes from functions SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ROOT_ENV_FILE="$PROJECT_ROOT/.env" -# Source environment variables if .env file exists -if [ -f "$SCRIPT_DIR/.env" ]; then +# Source environment variables if root .env file exists +if [ -f "$ROOT_ENV_FILE" ]; then set -a - source "$SCRIPT_DIR/.env" + source "$ROOT_ENV_FILE" set +a fi diff --git a/docker/deploy.sh b/deploy/docker/deploy.sh similarity index 81% rename from docker/deploy.sh rename to deploy/docker/deploy.sh index fbf3664b5..96cf621d8 100755 --- a/docker/deploy.sh +++ b/deploy/docker/deploy.sh @@ -10,11 +10,17 @@ fi set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +DEPLOY_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" CONST_FILE="$PROJECT_ROOT/backend/consts/const.py" DEPLOY_OPTIONS_FILE="$SCRIPT_DIR/deploy.options" -DEPLOYMENT_COMMON="$PROJECT_ROOT/scripts/deployment/common.sh" +DEPLOYMENT_COMMON="$DEPLOY_ROOT/common/common.sh" +VERSION_HELPER="$DEPLOY_ROOT/common/version.sh" ORIGINAL_ARGS=("$@") +ROOT_ENV_FILE="$PROJECT_ROOT/.env" +COMPOSE_DIR="$SCRIPT_DIR/compose" +DOCKER_ASSETS_DIR="$SCRIPT_DIR/assets" +SQL_DIR="$DEPLOY_ROOT/sql" if [ -f "$DEPLOYMENT_COMMON" ]; then # shellcheck source=/dev/null @@ -24,6 +30,11 @@ else exit 1 fi +if [ -f "$VERSION_HELPER" ]; then + # shellcheck source=/dev/null + source "$VERSION_HELPER" +fi + MODE_CHOICE_SAVED="" VERSION_CHOICE_SAVED="" IS_MAINLAND_SAVED="" @@ -34,18 +45,7 @@ APP_VERSION="" cd "$SCRIPT_DIR" -if [ ! -f ".env" ]; then - if [ -f ".env.example" ]; then - cp .env.example .env - echo "✅ Created docker/.env from docker/.env.example" - else - echo "❌ .env not found and .env.example is missing in $SCRIPT_DIR" - exit 1 - fi -fi - -set -a -source .env +deployment_source_root_env "$PROJECT_ROOT" "$PROJECT_ROOT/docker" || exit 1 # Parse arg MODE_CHOICE="" @@ -70,8 +70,11 @@ while [[ $# -gt 0 ]]; do echo " --components LIST" echo " --port-policy development|production" echo " --image-source general|mainland|local-latest" + echo " --version VERSION" echo " --use-local-config" echo " --reconfigure" + echo " --rotate-secrets" + echo " --refresh-es-key" echo " --config PATH" echo " --root-dir PATH" echo "" @@ -246,15 +249,15 @@ check_ports_in_env_files() { PORTS_TO_CHECK=() PORT_SOURCES=() - # Always include the main .env if present, plus any .env.* files + # Always include the root .env if present, plus image-source env variants. local env_files=() - if [ -f ".env" ]; then - env_files+=(".env") + if [ -f "$ROOT_ENV_FILE" ]; then + env_files+=("$ROOT_ENV_FILE") fi - # Include additional env variants such as .env.general and .env.mainland + # Include image-source env variants. local f - for f in .env.*; do + for f in "$DEPLOY_ROOT"/env/image-source.*.env; do if [ -f "$f" ]; then env_files+=("$f") fi @@ -408,11 +411,15 @@ trim_quotes() { } get_app_version() { + if declare -F deployment_read_version >/dev/null 2>&1; then + deployment_read_version "" + return 0 + fi + if [ ! -f "$CONST_FILE" ]; then echo "" return fi - local line line=$(grep -E 'APP_VERSION' "$CONST_FILE" | tail -n 1 || true) line="${line##*=}" @@ -436,16 +443,18 @@ persist_deploy_options() { } generate_minio_ak_sk() { - if [ -n "${MINIO_ACCESS_KEY:-}" ] && [ -n "${MINIO_SECRET_KEY:-}" ]; then - echo " Reusing existing MinIO access keys from docker/.env" + if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" != "true" ] && [ -n "${MINIO_ACCESS_KEY:-}" ] && [ -n "${MINIO_SECRET_KEY:-}" ]; then + echo " MinIO credentials unchanged; reusing root .env values" export MINIO_ACCESS_KEY export MINIO_SECRET_KEY - update_env_var "MINIO_ACCESS_KEY" "$MINIO_ACCESS_KEY" - update_env_var "MINIO_SECRET_KEY" "$MINIO_SECRET_KEY" return 0 fi - echo "🔑 Generating MinIO keys..." + if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" = "true" ]; then + echo "🔁 Rotating MinIO keys..." + else + echo "🔑 Generating missing MinIO keys..." + fi if [ "$(uname -s | tr '[:upper:]' '[:lower:]')" = "mingw" ] || [ "$(uname -s | tr '[:upper:]' '[:lower:]')" = "msys" ]; then # Windows @@ -493,40 +502,86 @@ generate_jwt() { } generate_supabase_keys() { - if [ "$DEPLOYMENT_VERSION" = "full" ]; then - # Function to generate Supabase secrets - echo "🔑 Generating Supabase keys..." + if [ "$DEPLOYMENT_VERSION" != "full" ]; then + return 0 + fi - # Generate fresh keys on every run for security - export JWT_SECRET=$(openssl rand -base64 32 | tr -d '[:space:]') - export SECRET_KEY_BASE=$(openssl rand -base64 64 | tr -d '[:space:]') - export VAULT_ENC_KEY=$(openssl rand -base64 32 | tr -d '[:space:]') + echo "🔑 Checking Supabase keys..." - # Generate JWT-dependent keys using the new JWT_SECRET - local anon_key=$(generate_jwt "anon") - local service_role_key=$(generate_jwt "service_role") + if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" != "true" ] \ + && [ -n "${JWT_SECRET:-}" ] \ + && [ -n "${SECRET_KEY_BASE:-}" ] \ + && [ -n "${VAULT_ENC_KEY:-}" ] \ + && [ -n "${SUPABASE_KEY:-}" ] \ + && [ -n "${SERVICE_ROLE_KEY:-}" ]; then + echo " Supabase secrets unchanged; reusing root .env values" + return 0 + fi - # Update or add all keys to the .env file + if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" = "true" ] || [ -z "${JWT_SECRET:-}" ]; then + export JWT_SECRET=$(openssl rand -base64 32 | tr -d '[:space:]') update_env_var "JWT_SECRET" "$JWT_SECRET" + fi + if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" = "true" ] || [ -z "${SECRET_KEY_BASE:-}" ]; then + export SECRET_KEY_BASE=$(openssl rand -base64 64 | tr -d '[:space:]') update_env_var "SECRET_KEY_BASE" "$SECRET_KEY_BASE" + fi + if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" = "true" ] || [ -z "${VAULT_ENC_KEY:-}" ]; then + export VAULT_ENC_KEY=$(openssl rand -base64 32 | tr -d '[:space:]') update_env_var "VAULT_ENC_KEY" "$VAULT_ENC_KEY" - update_env_var "SUPABASE_KEY" "$anon_key" - update_env_var "SERVICE_ROLE_KEY" "$service_role_key" + fi - # Reload the environment variables from the updated .env file - source .env - echo " ✅ Supabase keys generated successfully" + if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" = "true" ] || [ -z "${SUPABASE_KEY:-}" ]; then + SUPABASE_KEY=$(generate_jwt "anon") + export SUPABASE_KEY + update_env_var "SUPABASE_KEY" "$SUPABASE_KEY" + fi + if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" = "true" ] || [ -z "${SERVICE_ROLE_KEY:-}" ]; then + SERVICE_ROLE_KEY=$(generate_jwt "service_role") + export SERVICE_ROLE_KEY + update_env_var "SERVICE_ROLE_KEY" "$SERVICE_ROLE_KEY" + fi + + set -a + source "$ROOT_ENV_FILE" + set +a + if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" = "true" ]; then + echo " ✅ Supabase secrets rotated" + else + echo " ✅ Missing Supabase secrets generated" fi } +validate_elasticsearch_api_key() { + local api_key="$1" + local http_code + [ -n "$api_key" ] || return 1 + http_code=$(docker exec nexent-elasticsearch curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: ApiKey $api_key" \ + "http://localhost:9200/_security/_authenticate" 2>/dev/null || true) + [ "$http_code" = "200" ] +} generate_elasticsearch_api_key() { # Function to generate Elasticsearch API key wait_for_elasticsearch_healthy || { echo " ❌ Elasticsearch health check failed"; return 0; } + if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" != "true" ] \ + && [ "${DEPLOYMENT_REFRESH_ES_KEY:-false}" != "true" ] \ + && [ -n "${ELASTICSEARCH_API_KEY:-}" ]; then + echo "🔑 Validating existing ELASTICSEARCH_API_KEY..." + if validate_elasticsearch_api_key "$ELASTICSEARCH_API_KEY"; then + echo " ELASTICSEARCH_API_KEY unchanged; existing key is valid" + return 0 + fi + echo " Existing ELASTICSEARCH_API_KEY is invalid; generating a replacement" + elif [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" = "true" ] || [ "${DEPLOYMENT_REFRESH_ES_KEY:-false}" = "true" ]; then + echo "🔁 Refreshing ELASTICSEARCH_API_KEY by request..." + fi + # Generate API key echo "🔑 Generating ELASTICSEARCH_API_KEY..." - API_KEY_JSON=$(docker exec nexent-elasticsearch curl -s -u "elastic:$ELASTIC_PASSWORD" "http://localhost:9200/_security/api_key" -H "Content-Type: application/json" -d '{"name":"my_api_key","role_descriptors":{"my_role":{"cluster":["all"],"index":[{"names":["*"],"privileges":["all"]}]}}}') + API_KEY_JSON=$(docker exec nexent-elasticsearch curl -s -u "elastic:${ELASTIC_PASSWORD:-nexent@2025}" "http://localhost:9200/_security/api_key" -H "Content-Type: application/json" -d '{"name":"my_api_key","role_descriptors":{"my_role":{"cluster":["all"],"index":[{"names":["*"],"privileges":["all"]}]}}}') # Extract API key and add to .env ELASTICSEARCH_API_KEY=$(echo "$API_KEY_JSON" | grep -o '"encoded":"[^"]*"' | awk -F'"' '{print $4}') @@ -538,30 +593,30 @@ generate_elasticsearch_api_key() { generate_env_for_infrastructure() { # Function to generate complete environment file for infrastructure mode using generate_env.sh - echo "🔑 Updating docker/.env for infrastructure mode..." + echo "🔑 Updating root .env for infrastructure mode..." echo " 🚀 Running generate_env.sh..." # Check if generate_env.sh exists - if [ ! -f "generate_env.sh" ]; then - echo " ❌ ERROR generate_env.sh not found in docker directory" + if [ ! -f "$SCRIPT_DIR/generate_env.sh" ]; then + echo " ❌ ERROR generate_env.sh not found in deploy/docker directory" return 1 fi # Make sure the script is executable and run it - chmod +x generate_env.sh + chmod +x "$SCRIPT_DIR/generate_env.sh" # Export DEPLOYMENT_VERSION to ensure generate_env.sh can access it export DEPLOYMENT_VERSION - if ./generate_env.sh; then - echo " ✅ docker/.env updated successfully for infrastructure mode!" - if [ -f ".env" ]; then + if DEPLOYMENT_ROOT_ENV="$ROOT_ENV_FILE" bash "$SCRIPT_DIR/generate_env.sh"; then + echo " ✅ root .env updated successfully for infrastructure mode!" + if [ -f "$ROOT_ENV_FILE" ]; then set -a - source .env + source "$ROOT_ENV_FILE" set +a - echo " ✅ Environment variables loaded from docker/.env" + echo " ✅ Environment variables loaded from root .env" else - echo " ⚠️ Warning: docker/.env file not found after generation" + echo " ⚠️ Warning: root .env file not found after generation" return 1 fi else @@ -684,18 +739,17 @@ select_deployment_mode() { ROOT_DIR="$ROOT_DIR_PARAM" echo " 📁 Using ROOT_DIR from parameter: $ROOT_DIR" # Write to .env file - if grep -q "^ROOT_DIR=" .env; then + if grep -q "^ROOT_DIR=" "$ROOT_ENV_FILE"; then # Update existing ROOT_DIR in .env - sed -i "s|^ROOT_DIR=.*|ROOT_DIR=\"$ROOT_DIR\"|" .env + update_env_var "ROOT_DIR" "$ROOT_DIR" else # Add new ROOT_DIR to .env - echo "# Root dir" >> .env - echo "ROOT_DIR=\"$ROOT_DIR\"" >> .env + update_env_var "ROOT_DIR" "$ROOT_DIR" fi - elif grep -q "^ROOT_DIR=" .env; then + elif grep -q "^ROOT_DIR=" "$ROOT_ENV_FILE"; then # Check if ROOT_DIR already exists in .env (second priority) # Extract existing ROOT_DIR value from .env - env_root_dir=$(grep "^ROOT_DIR=" .env | cut -d'=' -f2 | sed 's/^"//;s/"$//') + env_root_dir=$(grep "^ROOT_DIR=" "$ROOT_ENV_FILE" | cut -d'=' -f2 | sed 's/^"//;s/"$//') ROOT_DIR="$env_root_dir" echo " 📁 Use existing ROOT_DIR path: $env_root_dir" @@ -705,8 +759,7 @@ select_deployment_mode() { read -p " 📁 Enter ROOT_DIR path (default: $default_root_dir): " user_root_dir ROOT_DIR="${user_root_dir:-$default_root_dir}" - echo "# Root dir" >> .env - echo "ROOT_DIR=\"$ROOT_DIR\"" >> .env + update_env_var "ROOT_DIR" "$ROOT_DIR" fi echo "" echo "--------------------------------" @@ -720,30 +773,19 @@ clean() { export COMPOSE_FILE_SUFFIX= export DEPLOYMENT_VERSION= - if [ -f ".env.bak" ]; then - rm .env.bak - fi + rm -f "$ROOT_ENV_FILE.bak" ".env.bak" } update_env_var() { - # Function to update or add a key-value pair to .env + # Function to update or add a key-value pair to root .env local key="$1" local value="$2" - local env_file=".env" - - # Ensure the .env file exists - touch "$env_file" - - if grep -q "^${key}=" "$env_file"; then - # Key exists, so update it. Escape \ and & for sed's replacement string. - # Use ~ as the separator to avoid issues with / in the value. - local escaped_value=$(echo "$value" | sed -e 's/\\/\\\\/g' -e 's/&/\\&/g') - sed -i.bak "s~^${key}=.*~${key}=\"${escaped_value}\"~" "$env_file" + deployment_update_env_var_file "$ROOT_ENV_FILE" "$key" "$value" + if [ "${DEPLOYMENT_LAST_ENV_WRITE_CHANGED:-false}" = "true" ]; then + echo " 📝 .env updated: $key" else - # Key doesn't exist, so add it - echo "${key}=\"${value}\"" >> "$env_file" + echo " ↺ .env unchanged: $key" fi - } create_dir_with_permission() { @@ -772,9 +814,35 @@ create_dir_with_permission() { fi } +sql_files_checksum() { + local payload="" + local file rel checksum + + if [ ! -d "$SQL_DIR" ]; then + echo "Error: SQL directory not found: $SQL_DIR" >&2 + return 1 + fi + + while IFS= read -r file; do + [ -n "$file" ] || continue + rel="${file#"$SQL_DIR/"}" + checksum="$(deployment_sha256_file "$file")" + payload="${payload}${rel}:${checksum}"$'\n' + done < <(find "$SQL_DIR" -type f -name '*.sql' -print | sort -V) + + deployment_sha256_string "$payload" +} + +update_sql_files_checksum() { + NEXENT_SQL_FILES_CHECKSUM="$(sql_files_checksum)" + export NEXENT_SQL_FILES_CHECKSUM + update_env_var "NEXENT_SQL_FILES_CHECKSUM" "$NEXENT_SQL_FILES_CHECKSUM" + echo " SQL files checksum: $NEXENT_SQL_FILES_CHECKSUM" +} + prepare_directory_and_data() { # Initialize the sql script permission - chmod 644 "init.sql" + chmod 644 "$SQL_DIR/init.sql" echo "🔧 Creating directory with permission..." create_dir_with_permission "$ROOT_DIR/elasticsearch" 775 @@ -782,12 +850,19 @@ prepare_directory_and_data() { create_dir_with_permission "$ROOT_DIR/minio" 775 create_dir_with_permission "$ROOT_DIR/redis" 775 - cp -rn volumes $ROOT_DIR + cp -rn "$DOCKER_ASSETS_DIR/volumes" "$ROOT_DIR" chmod -R 775 $ROOT_DIR/volumes echo " 📁 Directory $ROOT_DIR/volumes has been created and permissions set to 775." + mkdir -p "$ROOT_DIR/volumes/db/data" "$ROOT_DIR/volumes/db/init" + if [ -f "$SQL_DIR/supabase/init/data.sql" ]; then + cp -f "$SQL_DIR/supabase/init/data.sql" "$ROOT_DIR/volumes/db/init/data.sql" + fi + chmod -R 775 "$ROOT_DIR/volumes/db" + echo " Supabase data directory initialized; SQL files are mounted from $SQL_DIR/supabase." + # Copy sync_user_supabase2pg.py to ROOT_DIR for container access - cp -rn scripts $ROOT_DIR + cp -rn "$DOCKER_ASSETS_DIR/scripts" "$ROOT_DIR" chmod 644 "$ROOT_DIR/scripts/sync_user_supabase2pg.py" echo " 📁 update scripts copied to $ROOT_DIR" @@ -797,8 +872,8 @@ prepare_directory_and_data() { echo " 🖥️ Nexent user workspace: $NEXENT_USER_DIR" # Copy official-skills-zip folder to /mnt/nexent - if [ -d "official-skills-zip" ]; then - cp -rn official-skills-zip "$NEXENT_USER_DIR/" + if [ -d "$DOCKER_ASSETS_DIR/official-skills-zip" ]; then + cp -rn "$DOCKER_ASSETS_DIR/official-skills-zip" "$NEXENT_USER_DIR/" chmod -R 775 "$NEXENT_USER_DIR/official-skills-zip" echo " 📦 Official skills copied to $NEXENT_USER_DIR/official-skills-zip" else @@ -831,7 +906,7 @@ deploy_core_services() { fi echo "👀 Starting core services: ${core_services[*]}" - if ! ${docker_compose_command} -p nexent -f "docker-compose${COMPOSE_FILE_SUFFIX}" up -d "${core_services[@]}"; then + if ! ${docker_compose_command} --env-file "$ROOT_ENV_FILE" -p nexent -f "$COMPOSE_DIR/docker-compose${COMPOSE_FILE_SUFFIX}" up -d "${core_services[@]}"; then echo " ❌ ERROR Failed to start core services" return 1 fi @@ -840,12 +915,12 @@ deploy_core_services() { stop_unselected_data_process_service() { deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "data-process" && return 0 - local compose_file="docker-compose${COMPOSE_FILE_SUFFIX}" + local compose_file="$COMPOSE_DIR/docker-compose${COMPOSE_FILE_SUFFIX}" [ -f "$compose_file" ] || return 0 echo "data-process is not selected; stopping existing Docker container if present..." - ${docker_compose_command} -p nexent -f "$compose_file" stop nexent-data-process >/dev/null 2>&1 || true - ${docker_compose_command} -p nexent -f "$compose_file" rm -f nexent-data-process >/dev/null 2>&1 || true + ${docker_compose_command} --env-file "$ROOT_ENV_FILE" -p nexent -f "$compose_file" stop nexent-data-process >/dev/null 2>&1 || true + ${docker_compose_command} --env-file "$ROOT_ENV_FILE" -p nexent -f "$compose_file" rm -f nexent-data-process >/dev/null 2>&1 || true } deploy_infrastructure() { @@ -864,7 +939,7 @@ deploy_infrastructure() { fi if [ -n "$INFRA_SERVICES" ]; then - if ! ${docker_compose_command} -p nexent -f "docker-compose${COMPOSE_FILE_SUFFIX}" up -d $INFRA_SERVICES; then + if ! ${docker_compose_command} --env-file "$ROOT_ENV_FILE" -p nexent -f "$COMPOSE_DIR/docker-compose${COMPOSE_FILE_SUFFIX}" up -d $INFRA_SERVICES; then echo " ❌ ERROR Failed to start infrastructure services" return 1 fi @@ -881,13 +956,13 @@ deploy_infrastructure() { echo "" echo "🔧 Starting Supabase services..." # Check if the supabase compose file exists - if [ ! -f "docker-compose-supabase${COMPOSE_FILE_SUFFIX}" ]; then - echo " ❌ ERROR Supabase compose file not found: docker-compose-supabase${COMPOSE_FILE_SUFFIX}" + if [ ! -f "$COMPOSE_DIR/docker-compose-supabase${COMPOSE_FILE_SUFFIX}" ]; then + echo " ❌ ERROR Supabase compose file not found: $COMPOSE_DIR/docker-compose-supabase${COMPOSE_FILE_SUFFIX}" return 1 fi # Start Supabase services - if ! $docker_compose_command -p nexent -f "docker-compose-supabase${COMPOSE_FILE_SUFFIX}" up -d; then + if ! $docker_compose_command --env-file "$ROOT_ENV_FILE" -p nexent -f "$COMPOSE_DIR/docker-compose-supabase${COMPOSE_FILE_SUFFIX}" up -d; then echo " ❌ ERROR Failed to start supabase services" return 1 fi @@ -903,8 +978,8 @@ deploy_infrastructure() { deploy_monitoring() { deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring" || return 0 - if [ ! -f "docker-compose-monitoring.yml" ]; then - echo " ❌ ERROR Monitoring compose file not found: docker-compose-monitoring.yml" + if [ ! -f "$COMPOSE_DIR/docker-compose-monitoring.yml" ]; then + echo " ❌ ERROR Monitoring compose file not found: $COMPOSE_DIR/docker-compose-monitoring.yml" return 1 fi @@ -916,7 +991,7 @@ deploy_monitoring() { esac echo "🔭 Starting monitoring services..." - if ! ${docker_compose_command} "${profile_args[@]}" -f "docker-compose-monitoring.yml" up -d; then + if ! ${docker_compose_command} --env-file "$ROOT_ENV_FILE" "${profile_args[@]}" -f "$COMPOSE_DIR/docker-compose-monitoring.yml" up -d; then echo " ❌ ERROR Failed to start monitoring services" return 1 fi @@ -927,8 +1002,8 @@ configure_root_dir_from_env() { ROOT_DIR="$ROOT_DIR_PARAM" echo " 📁 Using ROOT_DIR from parameter: $ROOT_DIR" update_env_var "ROOT_DIR" "$ROOT_DIR" - elif grep -q "^ROOT_DIR=" .env; then - ROOT_DIR="$(grep "^ROOT_DIR=" .env | cut -d'=' -f2 | sed 's/^"//;s/"$//')" + elif grep -q "^ROOT_DIR=" "$ROOT_ENV_FILE"; then + ROOT_DIR="$(grep "^ROOT_DIR=" "$ROOT_ENV_FILE" | cut -d'=' -f2 | sed 's/^"//;s/"$//')" echo " 📁 Use existing ROOT_DIR path: $ROOT_DIR" else local default_root_dir="$HOME/nexent-data" @@ -982,11 +1057,11 @@ apply_deployment_common_config() { case "$DEPLOYMENT_REGISTRY_PROFILE" in mainland) IS_MAINLAND_SAVED="Y" - source .env.mainland + source "$DEPLOY_ROOT/env/image-source.mainland.env" ;; general|local-latest) IS_MAINLAND_SAVED="N" - source .env.general + source "$DEPLOY_ROOT/env/image-source.general.env" ;; esac @@ -1025,23 +1100,7 @@ select_deployment_version() { ;; esac - # Save the version choice to .env file - local key="DEPLOYMENT_VERSION" - local value="$DEPLOYMENT_VERSION" - local env_file=".env" - - # Ensure the .env file exists - touch "$env_file" - - if grep -q "^${key}=" "$env_file"; then - # Key exists, so update it. Escape \ and & for sed's replacement string. - # Use ~ as the separator to avoid issues with / in the value. - local escaped_value=$(echo "$value" | sed -e 's/\\/\\\\/g' -e 's/&/\\&/g') - sed -i.bak "s~^${key}=.*~${key}=\"${escaped_value}\"~" "$env_file" - else - # Key doesn't exist, so add it - echo "${key}=\"${value}\"" >> "$env_file" - fi + update_env_var "DEPLOYMENT_VERSION" "$DEPLOYMENT_VERSION" echo "" echo "--------------------------------" @@ -1054,8 +1113,8 @@ setup_package_install_script() { mkdir -p "openssh-server/config/custom-cont-init.d" # Copy the fixed installation script - if [ -f "openssh-install-script.sh" ]; then - cp "openssh-install-script.sh" "openssh-server/config/custom-cont-init.d/openssh-start-script" + if [ -f "$SCRIPT_DIR/openssh-install-script.sh" ]; then + cp "$SCRIPT_DIR/openssh-install-script.sh" "openssh-server/config/custom-cont-init.d/openssh-start-script" chmod +x "openssh-server/config/custom-cont-init.d/openssh-start-script" echo " ✅ Package installation script created/updated" else @@ -1068,7 +1127,7 @@ wait_for_elasticsearch_healthy() { # Function to wait for Elasticsearch to become healthy local retries=0 local max_retries=${1:-60} # Default 10 minutes, can be overridden - while ! ${docker_compose_command} -p nexent -f "docker-compose${COMPOSE_FILE_SUFFIX}" ps nexent-elasticsearch | grep -q "healthy" && [ $retries -lt $max_retries ]; do + while ! ${docker_compose_command} --env-file "$ROOT_ENV_FILE" -p nexent -f "$COMPOSE_DIR/docker-compose${COMPOSE_FILE_SUFFIX}" ps nexent-elasticsearch | grep -q "healthy" && [ $retries -lt $max_retries ]; do echo "⏳ Waiting for Elasticsearch to become healthy... (attempt $((retries + 1))/$max_retries)" sleep 10 retries=$((retries + 1)) @@ -1240,6 +1299,7 @@ prompt_super_admin_password() { echo "" >&2 echo "🔐 Super Admin User Password Setup" >&2 echo " Email: suadmin@nexent.com" >&2 + echo " Requirement: $(deployment_password_validation_message)" >&2 echo "" >&2 while [ $attempts -lt $max_attempts ]; do @@ -1255,6 +1315,12 @@ prompt_super_admin_password() { continue fi + if ! deployment_validate_password "$password"; then + echo " ❌ $(deployment_password_validation_message)" >&2 + attempts=$((attempts + 1)) + continue + fi + # Confirm password input echo " 🔐 Please confirm the password:" >&2 read -s password_confirm @@ -1347,12 +1413,12 @@ choose_image_env() { is_mainland=$(sanitize_input "$is_mainland") if [[ "$is_mainland" =~ ^[Yy]$ ]]; then IS_MAINLAND_SAVED="Y" - echo "🌐 Detected mainland China network, using .env.mainland for image sources." - source .env.mainland + echo "🌐 Detected mainland China network, using image-source.mainland.env for image sources." + source "$DEPLOY_ROOT/env/image-source.mainland.env" else IS_MAINLAND_SAVED="N" - echo "🌐 Using general image sources from .env.general." - source .env.general + echo "🌐 Using general image sources from image-source.general.env." + source "$DEPLOY_ROOT/env/image-source.general.env" fi echo "" @@ -1369,7 +1435,7 @@ main_deploy() { APP_VERSION="$(get_app_version)" if [ -z "$APP_VERSION" ]; then - echo "❌ Failed to get app version, please check the backend/consts/const.py file" + echo "❌ Failed to get app version, please check VERSION or backend/consts/const.py" exit 1 fi echo "🌐 App version: $APP_VERSION" @@ -1394,6 +1460,7 @@ main_deploy() { # Add permission prepare_directory_and_data || { echo "❌ Permission setup failed"; exit 1; } + update_sql_files_checksum || { echo "ERROR SQL checksum update failed"; exit 1; } generate_minio_ak_sk || { echo "❌ MinIO key generation failed"; exit 1; } @@ -1425,8 +1492,8 @@ main_deploy() { echo "🎉 Infrastructure deployment completed successfully!" echo " You can now start the core services manually using dev containers" - echo " Environment file available at: $SCRIPT_DIR/.env" - echo "💡 Use 'source docker/.env' from the project root to load environment variables" + echo " Environment file available at: $ROOT_ENV_FILE" + echo "💡 Use 'source .env' from the project root to load environment variables" # Pull MCP image for later use pull_mcp_image diff --git a/deploy/docker/generate_env.sh b/deploy/docker/generate_env.sh new file mode 100755 index 000000000..414c753a7 --- /dev/null +++ b/deploy/docker/generate_env.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +ENV_FILE="${DEPLOYMENT_ROOT_ENV:-$PROJECT_ROOT/.env}" +ENV_EXAMPLE="$PROJECT_ROOT/.env.example" +LEGACY_ENV="$PROJECT_ROOT/docker/.env" +LEGACY_ENV_EXAMPLE="$PROJECT_ROOT/docker/.env.example" + +if [ "${NEXENT_GENERATE_ENV_SKIP_MAIN:-false}" != "true" ]; then + echo " 📁 Target .env location: $ENV_FILE" +fi + +update_env_var() { + local key="$1" + local value="$2" + local escaped_value + local current_value + + touch "$ENV_FILE" + escaped_value=$(printf '%s' "$value" | sed -e 's/\\/\\\\/g' -e 's/&/\\&/g') + + if grep -q "^${key}=" "$ENV_FILE"; then + current_value="$(grep "^${key}=" "$ENV_FILE" | tail -n 1 | cut -d'=' -f2- | sed 's/[[:space:]]*$//;s/^"//;s/"$//;s/^'\''//;s/'\''$//')" + if [ "$current_value" = "$value" ]; then + echo " ↺ root .env unchanged: $key" + return 0 + fi + sed -i.bak "s~^${key}=.*~${key}=${escaped_value}~" "$ENV_FILE" + rm -f "${ENV_FILE}.bak" + else + printf '%s=%s\n' "$key" "$value" >> "$ENV_FILE" + fi + echo " 📝 root .env updated: $key" +} + +# Function to copy and prepare .env file +prepare_env_file() { + echo " 📝 Preparing root .env file..." + + if [ -f "$ENV_FILE" ]; then + echo " ✅ Using existing root .env" + elif [ -f "$LEGACY_ENV" ]; then + echo " root .env not found, copying docker/.env..." + cp "$LEGACY_ENV" "$ENV_FILE" + echo " Created root .env from docker/.env" + elif [ -f "$ENV_EXAMPLE" ]; then + echo " 📋 root .env not found, copying .env.example..." + cp "$ENV_EXAMPLE" "$ENV_FILE" + echo " ✅ Created root .env from .env.example" + elif [ -f "$LEGACY_ENV_EXAMPLE" ]; then + echo " 📋 root .env not found, copying docker/.env.example..." + cp "$LEGACY_ENV_EXAMPLE" "$ENV_FILE" + echo " ✅ Created root .env from docker/.env.example" + else + echo " ERROR Neither root .env nor docker/.env nor .env.example exists" + ERROR_OCCURRED=1 + return 1 + fi +} + +# Function to update .env file with generated keys +update_env_file() { + echo " 📝 Updating root .env file with generated keys..." + + if [ ! -f "$ENV_FILE" ]; then + echo " ❌ ERROR root .env file does not exist" + ERROR_OCCURRED=1 + return 1 + fi + + update_env_var "MINIO_ACCESS_KEY" "$MINIO_ACCESS_KEY" + update_env_var "MINIO_SECRET_KEY" "$MINIO_SECRET_KEY" + + if [ -n "$ELASTICSEARCH_API_KEY" ]; then + update_env_var "ELASTICSEARCH_API_KEY" "$ELASTICSEARCH_API_KEY" + fi + + if [ -n "$SSH_USERNAME" ]; then + update_env_var "SSH_USERNAME" "$SSH_USERNAME" + fi + + if [ -n "$SSH_PASSWORD" ]; then + update_env_var "SSH_PASSWORD" "$SSH_PASSWORD" + fi + echo " ✅ Generated keys updated successfully" + + # Force update development environment service URLs for localhost access + echo " 🔧 Updating service URLs for localhost development environment..." + + update_env_var "ELASTICSEARCH_HOST" "http://localhost:9210" + update_env_var "CONFIG_SERVICE_URL" "http://localhost:5010" + update_env_var "RUNTIME_SERVICE_URL" "http://localhost:5014" + update_env_var "ELASTICSEARCH_SERVICE" "http://localhost:5010/api" + update_env_var "NEXENT_MCP_SERVER" "http://localhost:5011" + update_env_var "DATA_PROCESS_SERVICE" "http://localhost:5012/api" + update_env_var "NORTHBOUND_API_SERVER" "http://localhost:5013/api" + update_env_var "MCP_MANAGEMENT_API" "http://localhost:5015" + update_env_var "MINIO_ENDPOINT" "http://localhost:9010" + update_env_var "REDIS_URL" "redis://localhost:6379/0" + update_env_var "REDIS_BACKEND_URL" "redis://localhost:6379/1" + update_env_var "POSTGRES_HOST" "localhost" + update_env_var "POSTGRES_PORT" "5434" + + # Supabase Configuration (Only for full version) + if [ "$DEPLOYMENT_VERSION" = "full" ]; then + if [ -n "$SUPABASE_KEY" ]; then + update_env_var "SUPABASE_KEY" "$SUPABASE_KEY" + fi + + if [ -n "$SERVICE_ROLE_KEY" ]; then + update_env_var "SERVICE_ROLE_KEY" "$SERVICE_ROLE_KEY" + fi + + update_env_var "SUPABASE_URL" "http://localhost:8000" + update_env_var "API_EXTERNAL_URL" "http://localhost:8000" + update_env_var "SITE_URL" "http://localhost:3011" + fi + + echo " ✅ root .env updated successfully with localhost development URLs" +} + +# Function to show summary +show_summary() { + echo "🎉 Environment generation completed!" + + echo "" + echo "--------------------------------" + echo "" + + echo "🔣 Generated keys:" + echo " 🔑 MINIO_ACCESS_KEY: $MINIO_ACCESS_KEY" + echo " 🔑 MINIO_SECRET_KEY: $MINIO_SECRET_KEY" + if [ -n "$ELASTICSEARCH_API_KEY" ]; then + echo " 🔑 ELASTICSEARCH_API_KEY: $ELASTICSEARCH_API_KEY" + else + echo " ⚠️ ELASTICSEARCH_API_KEY: Not generated (Elasticsearch not available)" + fi + if [ -n "$SUPABASE_KEY" ]; then + echo " 🔑 SUPABASE_KEY: $SUPABASE_KEY" + fi + if [ -n "$SERVICE_ROLE_KEY" ]; then + echo " 🔑 SERVICE_ROLE_KEY: $SERVICE_ROLE_KEY" + fi + if [ -n "$SSH_USERNAME" ]; then + echo " 👤 SSH_USERNAME: $SSH_USERNAME" + fi + if [ -n "$SSH_PASSWORD" ]; then + echo " 🔑 SSH_PASSWORD: [HIDDEN]" + fi + if [ -z "$ELASTICSEARCH_API_KEY" ]; then + echo " ⚠️ Note: To generate ELASTICSEARCH_API_KEY later, please:" + echo " 1. Start Elasticsearch: docker-compose -p nexent up -d nexent-elasticsearch" + echo " 2. Wait for it to become healthy" + echo " 3. Run this script again or manually generate the API key" + fi +} + +# Main execution +main() { + # Step 1: Prepare .env file + prepare_env_file || { echo "❌ Failed to prepare .env file"; exit 1; } + + # Step 2: Update .env file + echo "" + update_env_file || { echo "❌ Failed to update .env file"; exit 1; } + + # Step 3: Show summary + show_summary +} + +# Run main function +if [ "${NEXENT_GENERATE_ENV_SKIP_MAIN:-false}" != "true" ]; then + main "$@" +fi diff --git a/docker/openssh-install-script.sh b/deploy/docker/openssh-install-script.sh similarity index 100% rename from docker/openssh-install-script.sh rename to deploy/docker/openssh-install-script.sh diff --git a/docker/start-monitoring.sh b/deploy/docker/start-monitoring.sh similarity index 96% rename from docker/start-monitoring.sh rename to deploy/docker/start-monitoring.sh index 48ca6cd3f..2032b24f5 100755 --- a/docker/start-monitoring.sh +++ b/deploy/docker/start-monitoring.sh @@ -8,8 +8,8 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MONITORING_DIR="$SCRIPT_DIR/monitoring" -COMPOSE_FILE="$SCRIPT_DIR/docker-compose-monitoring.yml" +MONITORING_DIR="$SCRIPT_DIR/assets/monitoring" +COMPOSE_FILE="$SCRIPT_DIR/compose/docker-compose-monitoring.yml" SUPPORTED_STACKS="otlp, collector, phoenix, langfuse, langsmith, grafana, zipkin" @@ -231,17 +231,17 @@ configure_stack() { case "$LOCAL_STACK" in collector) BACKEND_MONITORING_PROVIDER="otlp" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-./monitoring/otel-collector-config.yml}" + OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-config.yml}" COMPOSE_PROFILES=() ;; phoenix) BACKEND_MONITORING_PROVIDER="phoenix" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-./monitoring/otel-collector-phoenix-config.yml}" + OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-phoenix-config.yml}" COMPOSE_PROFILES=(--profile phoenix) ;; langfuse) BACKEND_MONITORING_PROVIDER="langfuse" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-./monitoring/otel-collector-langfuse-config.yml}" + OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-langfuse-config.yml}" COMPOSE_PROFILES=(--profile langfuse) LANGFUSE_INIT_PROJECT_PUBLIC_KEY="${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-pk-lf-nexent-local}" LANGFUSE_INIT_PROJECT_SECRET_KEY="${LANGFUSE_INIT_PROJECT_SECRET_KEY:-sk-lf-nexent-local}" @@ -252,7 +252,7 @@ configure_stack() { ;; langsmith) BACKEND_MONITORING_PROVIDER="langsmith" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-./monitoring/otel-collector-langsmith-config.yml}" + OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-langsmith-config.yml}" COMPOSE_PROFILES=() LANGSMITH_OTLP_TRACES_ENDPOINT="${LANGSMITH_OTLP_TRACES_ENDPOINT:-https://api.smith.langchain.com/otel/v1/traces}" LANGSMITH_PROJECT="${LANGSMITH_PROJECT:-nexent}" @@ -265,12 +265,12 @@ configure_stack() { ;; grafana) BACKEND_MONITORING_PROVIDER="grafana" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-./monitoring/otel-collector-grafana-config.yml}" + OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-grafana-config.yml}" COMPOSE_PROFILES=(--profile grafana) ;; zipkin) BACKEND_MONITORING_PROVIDER="zipkin" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-./monitoring/otel-collector-zipkin-config.yml}" + OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-zipkin-config.yml}" COMPOSE_PROFILES=(--profile zipkin) ;; esac @@ -356,8 +356,8 @@ print_access_hints() { print_backend_hints() { echo "" echo "🔧 To enable monitoring in your Nexent backend:" - echo " 1. Set ENABLE_TELEMETRY=true in docker/.env" - echo " 2. Set MONITORING_PROVIDER=$BACKEND_MONITORING_PROVIDER in docker/.env" + echo " 1. Set ENABLE_TELEMETRY=true in the project root .env" + echo " 2. Set MONITORING_PROVIDER=$BACKEND_MONITORING_PROVIDER in the project root .env" echo " 3. Set OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 for Docker services" echo " or http://localhost:${OTEL_COLLECTOR_HTTP_PORT:-4318} for a backend running on the host" echo " 4. Set MONITORING_DASHBOARD_URL as shown above when a UI is available" diff --git a/docker/uninstall.sh b/deploy/docker/uninstall.sh similarity index 73% rename from docker/uninstall.sh rename to deploy/docker/uninstall.sh index 801a9f4f7..fe29dfec4 100755 --- a/docker/uninstall.sh +++ b/deploy/docker/uninstall.sh @@ -8,6 +8,9 @@ fi set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +ROOT_ENV_FILE="$PROJECT_ROOT/.env" +COMPOSE_DIR="$SCRIPT_DIR/compose" cd "$SCRIPT_DIR" DELETE_VOLUMES="" @@ -78,17 +81,17 @@ while [[ $# -gt 0 ]]; do esac done -if [ -f ".env" ]; then +if [ -f "$ROOT_ENV_FILE" ]; then set -a # shellcheck source=/dev/null - source .env + source "$ROOT_ENV_FILE" set +a fi -if [ -f ".env.generated" ]; then +if [ -f "$SCRIPT_DIR/.env.generated" ]; then set -a # shellcheck source=/dev/null - source .env.generated + source "$SCRIPT_DIR/.env.generated" set +a fi @@ -162,10 +165,35 @@ resolve_delete_volumes() { [[ "$answer" =~ ^[Yy]$ ]] } +remove_docker_named_volumes() { + command -v docker >/dev/null 2>&1 || return 0 + + local volume_names + volume_names="$(docker volume ls --format '{{.Name}}' 2>/dev/null || true)" + [ -n "$volume_names" ] || return 0 + + local volumes_to_remove=() + local volume + while IFS= read -r volume; do + [ -n "$volume" ] || continue + case "$volume" in + nexent_*|nexent-*|monitor_*) + volumes_to_remove+=("$volume") + ;; + esac + done <<< "$volume_names" + + if [ "${#volumes_to_remove[@]}" -gt 0 ]; then + echo "🧹 Removing Docker volumes: ${volumes_to_remove[*]}" + docker volume rm -f "${volumes_to_remove[@]}" >/dev/null 2>&1 || true + fi +} + docker_compose_down_file() { local compose_file="$1" local use_project_name="$2" local remove_volumes="$3" + local env_file_args=() [ -f "$compose_file" ] || return 0 @@ -173,16 +201,20 @@ docker_compose_down_file() { if [ "$remove_volumes" = "true" ]; then volume_args=(-v) fi + if [ -f "$ROOT_ENV_FILE" ]; then + env_file_args=(--env-file "$ROOT_ENV_FILE") + fi if [ "$use_project_name" = "true" ]; then - $docker_compose_command -p nexent -f "$compose_file" down --remove-orphans "${volume_args[@]}" || true + $docker_compose_command "${env_file_args[@]}" -p nexent -f "$compose_file" down --remove-orphans "${volume_args[@]}" || true else - $docker_compose_command -f "$compose_file" down --remove-orphans "${volume_args[@]}" || true + $docker_compose_command "${env_file_args[@]}" -f "$compose_file" down --remove-orphans "${volume_args[@]}" || true fi } remove_nexent_data_dirs() { local root_dir="${ROOT_DIR:-$HOME/nexent-data}" + local work_dir="$HOME/nexent" root_dir="${root_dir%/}" if [ -z "$root_dir" ] || [ "$root_dir" = "/" ]; then @@ -198,6 +230,8 @@ remove_nexent_data_dirs() { "$root_dir/volumes" "$root_dir/openssh-server" "$root_dir/scripts" + "$root_dir/skills" + "$work_dir" ) local dir @@ -224,13 +258,14 @@ main() { echo "ℹ️ Data volumes will be preserved." fi - docker_compose_down_file "docker-compose-monitoring.yml" false "$remove_volumes" - docker_compose_down_file "docker-compose-supabase.prod.yml" true "$remove_volumes" - docker_compose_down_file "docker-compose-supabase.yml" true "$remove_volumes" - docker_compose_down_file "docker-compose.prod.yml" true "$remove_volumes" - docker_compose_down_file "docker-compose.yml" true "$remove_volumes" + docker_compose_down_file "$COMPOSE_DIR/docker-compose-monitoring.yml" false "$remove_volumes" + docker_compose_down_file "$COMPOSE_DIR/docker-compose-supabase.prod.yml" true "$remove_volumes" + docker_compose_down_file "$COMPOSE_DIR/docker-compose-supabase.yml" true "$remove_volumes" + docker_compose_down_file "$COMPOSE_DIR/docker-compose.prod.yml" true "$remove_volumes" + docker_compose_down_file "$COMPOSE_DIR/docker-compose.yml" true "$remove_volumes" if [ "$remove_volumes" = "true" ]; then + remove_docker_named_volumes remove_nexent_data_dirs fi diff --git a/deploy/docker/upgrade.sh b/deploy/docker/upgrade.sh new file mode 100755 index 000000000..8ce1e7b47 --- /dev/null +++ b/deploy/docker/upgrade.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cat <<'NOTICE' +[WARN] docker/upgrade.sh is deprecated. +[WARN] Use deploy/docker/deploy.sh for both first install and upgrade. +[WARN] This compatibility wrapper does not delete Docker volumes. +NOTICE + +exec bash "$SCRIPT_DIR/deploy.sh" "$@" diff --git a/docker/.env.general b/deploy/env/image-source.general.env similarity index 100% rename from docker/.env.general rename to deploy/env/image-source.general.env diff --git a/docker/.env.mainland b/deploy/env/image-source.mainland.env similarity index 100% rename from docker/.env.mainland rename to deploy/env/image-source.mainland.env diff --git a/deploy/images/build.sh b/deploy/images/build.sh new file mode 100755 index 000000000..8a7459910 --- /dev/null +++ b/deploy/images/build.sh @@ -0,0 +1,459 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +VERSION_HELPER="$PROJECT_ROOT/deploy/common/version.sh" +DEPLOYMENT_COMMON="$PROJECT_ROOT/deploy/common/common.sh" +DOCKERFILE_DIR="$SCRIPT_DIR/dockerfiles" + +# shellcheck source=/dev/null +source "$VERSION_HELPER" +# shellcheck source=/dev/null +source "$DEPLOYMENT_COMMON" + +IMAGE="all" +IMAGES="" +COMPONENTS="" +PLATFORM="" +VERSION="$(deployment_read_version)" +REGISTRY="general" +DEPENDENCY_VARIANT="cpu" +TERMINAL_VARIANT="slim" +PUSH=false +LOAD=false +DRY_RUN=false +INTERACTIVE=false +ARGS_COUNT=$# +REQUESTED_IMAGES=() + +if [ "$ARGS_COUNT" -eq 0 ] && [ -t 0 ]; then + INTERACTIVE=true +fi + +usage() { + cat <<'USAGE' +Usage: deploy/images/build.sh [options] + +Options: + --images LIST Comma-separated image list: all,main,web,data-process,mcp,terminal,docs + --image IMAGE Compatibility alias for --images with one image + --all Build all images + --main Build nexent/nexent + --web Build nexent/nexent-web + --data-process Build nexent/nexent-data-process + --mcp Build nexent/nexent-mcp + --terminal Build nexent/nexent-ubuntu-terminal + --docs Build nexent/nexent-docs + --components LIST Compatibility mapping from deployment components to images. + --platform linux/amd64|linux/arm64|linux/amd64,linux/arm64 + --version VERSION Image tag, for example v2.2.1 or latest. Defaults to root VERSION. + --registry general|mainland + --dependency-variant cpu|gpu + data-process dependency variant. Defaults to cpu. + --terminal-variant slim|conda + terminal image variant. Defaults to slim. + --push + --load + --dry-run + --interactive Prompt for images, version, and registry. +USAGE +} + +while [ $# -gt 0 ]; do + case "$1" in + --image) IMAGE="$2"; shift 2 ;; + --images) IMAGES="$2"; shift 2 ;; + --all) REQUESTED_IMAGES=(all); shift ;; + --main) REQUESTED_IMAGES+=("main"); shift ;; + --web) REQUESTED_IMAGES+=("web"); shift ;; + --data-process) REQUESTED_IMAGES+=("data-process"); shift ;; + --mcp) REQUESTED_IMAGES+=("mcp"); shift ;; + --terminal) REQUESTED_IMAGES+=("terminal"); shift ;; + --docs) REQUESTED_IMAGES+=("docs"); shift ;; + --components) COMPONENTS="$2"; shift 2 ;; + --platform) PLATFORM="$2"; shift 2 ;; + --version) VERSION="$2"; shift 2 ;; + --registry) REGISTRY="$2"; shift 2 ;; + --dependency-variant|--data-process-dependency-variant) DEPENDENCY_VARIANT="$2"; shift 2 ;; + --terminal-variant) TERMINAL_VARIANT="$2"; shift 2 ;; + --push) PUSH=true; shift ;; + --load) LOAD=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --interactive) INTERACTIVE=true; shift ;; + --help|-h) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + esac +done + +prompt_choice() { + local prompt="$1" + local default_value="$2" + local value + read -r -p "$prompt" value || value="" + printf '%s' "${value:-$default_value}" +} + +add_image_if_missing() { + local image="$1" + local existing + for existing in "${SELECTED_IMAGES[@]}"; do + [ "$existing" = "$image" ] && return 0 + done + SELECTED_IMAGES+=("$image") +} + +select_all_images() { + SELECTED_IMAGES=(main web data-process mcp terminal docs) +} + +select_images_from_csv() { + local images="$1" + local old_ifs="$IFS" + local image normalized + + SELECTED_IMAGES=() + IFS=',' + for image in $images; do + normalized="$(deployment_trim "$image")" + case "$normalized" in + "" ) + ;; + all) + select_all_images + ;; + main|web|data-process|mcp|terminal|docs) + add_image_if_missing "$normalized" + ;; + *) + echo "Unsupported image: $normalized" >&2 + exit 1 + ;; + esac + done + IFS="$old_ifs" +} + +image_tui_multiselect() { + [ -t 0 ] || return 1 + + local images=(main web data-process mcp terminal docs) + local details=( + "backend API service" + "Next.js frontend" + "document parsing and vectorization worker" + "MCP proxy image" + "OpenSSH terminal tool image" + "VitePress documentation site" + ) + local selected=(1 1 0 0 0 0) + local cursor=0 + local i key key_tail selection + + image_tui_render() { + printf '\033[2J\033[H' + printf 'Select images to build\n' + printf 'Use Up/Down or j/k to move, Space to toggle, Enter to confirm, q to quit.\n\n' + local row marker check + for row in "${!images[@]}"; do + marker=" " + [ "$row" -eq "$cursor" ] && marker=">" + check=" " + [ "${selected[$row]}" = "1" ] && check="*" + printf '%s [%s] %s - %s\n' "$marker" "$check" "${images[$row]}" "${details[$row]}" + done + } + + printf '\033[?25l' + while true; do + image_tui_render + IFS= read -rsn1 key || key="" + if [ -z "$key" ]; then + selection="" + for i in "${!images[@]}"; do + if [ "${selected[$i]}" = "1" ]; then + selection="$(deployment_join_csv "$selection" "${images[$i]}")" + fi + done + if [ -n "$selection" ]; then + IMAGES="$selection" + break + fi + continue + fi + + if [ "$key" = $'\033' ]; then + IFS= read -rsn2 -t 0.1 key_tail || key_tail="" + key="${key}${key_tail}" + fi + + case "$key" in + $'\033[A'|k|K) + cursor=$((cursor - 1)) + [ "$cursor" -lt 0 ] && cursor=$((${#images[@]} - 1)) + ;; + $'\033[B'|j|J) + cursor=$((cursor + 1)) + [ "$cursor" -ge "${#images[@]}" ] && cursor=0 + ;; + " ") + if [ "${selected[$cursor]}" = "1" ]; then + selected[$cursor]=0 + else + selected[$cursor]=1 + fi + ;; + q|Q) + printf '\033[?25h' + printf '\033[2J\033[H' + echo "Image build configuration cancelled." >&2 + return 130 + ;; + esac + done + printf '\033[?25h' + printf '\033[2J\033[H' +} + +run_interactive_configuration() { + local root_version + root_version="$(deployment_read_version)" + + echo "Nexent image build configuration" + echo "" + + if [ -z "$IMAGES" ] && [ "${#REQUESTED_IMAGES[@]}" -eq 0 ] && [ -z "$COMPONENTS" ] && [ "$IMAGE" = "all" ]; then + if [ -t 0 ]; then + image_tui_multiselect || return $? + else + echo "Images:" + echo " main, web, data-process, mcp, terminal, docs" + IMAGES="$(prompt_choice "Enter images (default: main,web): " "main,web")" + fi + fi + + echo "Image version:" + echo " 1) latest" + echo " 2) Root VERSION ($root_version)" + local version_choice + version_choice="$(prompt_choice "Choose version [1/2] (default: 1): " "1")" + case "$version_choice" in + 1|latest|"") VERSION="latest" ;; + 2|root|version|VERSION) VERSION="$root_version" ;; + *) echo "Unsupported version choice: $version_choice" >&2; exit 1 ;; + esac + + echo "" + echo "Image registry:" + echo " 1) general (nexent/*)" + echo " 2) mainland (ccr.ccs.tencentyun.com/nexent-hub/*)" + local registry_choice + registry_choice="$(prompt_choice "Choose registry [1/2] (default: 1): " "1")" + case "$registry_choice" in + 2|mainland) REGISTRY="mainland" ;; + 1|general|"") REGISTRY="general" ;; + *) REGISTRY="$registry_choice" ;; + esac + +} + +if [ "$INTERACTIVE" = true ]; then + run_interactive_configuration +fi + +case "$REGISTRY" in + general) + REPO_PREFIX="nexent" + PY_MIRROR_ARGS=() + WEB_MIRROR_ARGS=() + ;; + mainland) + REPO_PREFIX="ccr.ccs.tencentyun.com/nexent-hub" + PY_MIRROR_ARGS=(--build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua) + WEB_MIRROR_ARGS=(--build-arg MIRROR=https://registry.npmmirror.com --build-arg APK_MIRROR=tsinghua) + ;; + *) echo "Unsupported registry: $REGISTRY" >&2; exit 1 ;; +esac + +case "$DEPENDENCY_VARIANT" in + cpu|gpu) ;; + *) echo "Unsupported data-process dependency variant: $DEPENDENCY_VARIANT" >&2; exit 1 ;; +esac + +case "$TERMINAL_VARIANT" in + slim|conda) ;; + *) echo "Unsupported terminal variant: $TERMINAL_VARIANT" >&2; exit 1 ;; +esac + +run_cmd() { + printf '+' + printf ' %q' "$@" + printf '\n' + if [ "$DRY_RUN" != true ]; then + "$@" + fi +} + +model_assets_complete() { + local model_assets_dir="$1" + + [ -f "$model_assets_dir/clip-vit-base-patch32/config.json" ] && \ + [ -d "$model_assets_dir/nltk_data" ] && \ + [ -d "$model_assets_dir/table-transformer-structure-recognition" ] && \ + [ -d "$model_assets_dir/yolox" ] +} + +prepare_model_assets() { + [ "$DRY_RUN" = true ] && return 0 + + local project_model_assets="$PROJECT_ROOT/model-assets" + local home_model_assets="${HOME:-}/model-assets" + local model_assets_repo="${MODEL_ASSETS_REPO:-}" + local tmp_model_assets + + if model_assets_complete "$project_model_assets"; then + echo "Using existing model-assets at $project_model_assets" + return 0 + fi + + if [ -n "${HOME:-}" ] && model_assets_complete "$home_model_assets"; then + echo "Copying cached model-assets from $home_model_assets" + mkdir -p "$project_model_assets" + cp -R "$home_model_assets"/. "$project_model_assets"/ + return 0 + fi + + command -v git >/dev/null 2>&1 || { + echo "git is required to clone model-assets for data-process builds." >&2 + exit 1 + } + git lfs version >/dev/null 2>&1 || { + echo "git-lfs is required to pull model-assets for data-process builds." >&2 + exit 1 + } + + if [ -z "$model_assets_repo" ]; then + if [ "$REGISTRY" = "mainland" ]; then + model_assets_repo="https://hf-mirror.com/Nexent-AI/model-assets" + else + model_assets_repo="https://huggingface.co/Nexent-AI/model-assets" + fi + fi + + tmp_model_assets="$PROJECT_ROOT/model-assets.tmp.$$" + echo "Cloning model-assets from $model_assets_repo" + rm -rf "$tmp_model_assets" + GIT_LFS_SKIP_SMUDGE=1 git clone "$model_assets_repo" "$tmp_model_assets" + ( + cd "$tmp_model_assets" + GIT_TRACE=1 GIT_CURL_VERBOSE=1 GIT_LFS_LOG=debug git lfs pull + rm -rf .git .gitattributes + ) + mkdir -p "$project_model_assets" + cp -R "$tmp_model_assets"/. "$project_model_assets"/ + rm -rf "$tmp_model_assets" +} + +build_one() { + local name="$1" + local dockerfile="$2" + shift 2 + local tag="$REPO_PREFIX/$name:$VERSION" + local cmd=(docker buildx build) + if [ -n "$PLATFORM" ]; then + cmd+=(--platform "$PLATFORM") + fi + cmd+=(-t "$tag" -f "$dockerfile") + if [ "$PUSH" = true ]; then + cmd+=(--push) + elif [ "$LOAD" = true ]; then + cmd+=(--load) + fi + cmd+=("$@" "$PROJECT_ROOT") + run_cmd "${cmd[@]}" +} + +build_selected_image() { + case "$1" in + main) build_one nexent "$DOCKERFILE_DIR/main/Dockerfile" "${PY_MIRROR_ARGS[@]}" ;; + web) build_one nexent-web "$DOCKERFILE_DIR/web/Dockerfile" "${WEB_MIRROR_ARGS[@]}" ;; + docs) build_one nexent-docs "$DOCKERFILE_DIR/docs/Dockerfile" "${WEB_MIRROR_ARGS[@]}" ;; + data-process) + local image_name="nexent-data-process" + [ "$DEPENDENCY_VARIANT" = "gpu" ] && image_name="${image_name}-gpu" + prepare_model_assets + build_one "$image_name" "$DOCKERFILE_DIR/data-process/Dockerfile" \ + --build-arg DATA_PROCESS_DEPENDENCY_VARIANT="$DEPENDENCY_VARIANT" \ + "${PY_MIRROR_ARGS[@]}" + ;; + mcp) build_one nexent-mcp "$DOCKERFILE_DIR/mcp/Dockerfile" "${PY_MIRROR_ARGS[@]}" ;; + terminal) + local image_name="nexent-ubuntu-terminal" + [ "$TERMINAL_VARIANT" = "conda" ] && image_name="nexent-ubuntu-terminal-conda" + build_one "$image_name" "$DOCKERFILE_DIR/terminal/Dockerfile" --build-arg TERMINAL_VARIANT="$TERMINAL_VARIANT" + ;; + *) echo "Unsupported image: $1" >&2; exit 1 ;; + esac +} + +select_images_from_components() { + local components="$1" + local old_ifs="$IFS" + local component normalized + + SELECTED_IMAGES=() + IFS=',' + for component in $components; do + normalized="$(deployment_trim "$component")" + case "$normalized" in + ""|infrastructure|supabase|monitoring) + ;; + application) + add_image_if_missing main + add_image_if_missing web + add_image_if_missing mcp + ;; + data-process) + add_image_if_missing data-process + ;; + terminal) + add_image_if_missing terminal + ;; + *) + echo "Unsupported component for image build: $normalized" >&2 + exit 1 + ;; + esac + done + IFS="$old_ifs" +} + +select_images_from_image_arg() { + SELECTED_IMAGES=() + if [ "$IMAGE" = "all" ]; then + select_all_images + else + select_images_from_csv "$IMAGE" + fi +} + +SELECTED_IMAGES=() +if [ "${#REQUESTED_IMAGES[@]}" -gt 0 ]; then + select_images_from_csv "$(deployment_join_csv "${REQUESTED_IMAGES[@]}")" +elif [ -n "$IMAGES" ]; then + select_images_from_csv "$IMAGES" +elif [ -n "$COMPONENTS" ]; then + select_images_from_components "$COMPONENTS" +else + select_images_from_image_arg +fi + +if [ "${#SELECTED_IMAGES[@]}" -eq 0 ]; then + echo "No Nexent images selected for build." + exit 0 +fi + +for selected in "${SELECTED_IMAGES[@]}"; do + build_selected_image "$selected" +done diff --git a/deploy/images/dockerfiles/data-process/Dockerfile b/deploy/images/dockerfiles/data-process/Dockerfile new file mode 100644 index 000000000..6881bc093 --- /dev/null +++ b/deploy/images/dockerfiles/data-process/Dockerfile @@ -0,0 +1,188 @@ +# syntax=docker/dockerfile:1.7 + +ARG DATA_PROCESS_DEPENDENCY_VARIANT=cpu + +FROM python:3.11-slim AS data-process-base +ARG MIRROR +ARG APT_MIRROR +ARG TARGETARCH +LABEL authors="nexent" + +# Set correct permissions as root +USER root + +# Configure apt sources based on build argument +RUN --mount=type=cache,id=nexent-data-process-apt-cache-${TARGETARCH},target=/var/cache/apt,sharing=locked \ + --mount=type=cache,id=nexent-data-process-apt-lists-${TARGETARCH},target=/var/lib/apt/lists,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + mkdir -p /var/cache/apt/archives /var/lib/apt/lists/partial && \ + if [ "$APT_MIRROR" = "tsinghua" ]; then \ + rm -f /etc/apt/sources.list.d/* && \ + echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list && \ + echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \ + echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-backports main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \ + echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bookworm-security main contrib non-free non-free-firmware" >> /etc/apt/sources.list; \ + fi && \ + apt-get update && \ + apt-get install -y --no-install-recommends --fix-missing \ + curl \ + postgresql-client \ + libmagic1 \ + libmagic-dev \ + libgl1 \ + coreutils && \ + apt-get autoremove -y && \ + rm -rf /tmp/* /var/tmp/* + +FROM data-process-base AS data-process-deps +ARG MIRROR +ARG TARGETARCH +ARG DATA_PROCESS_DEPENDENCY_VARIANT + +RUN --mount=type=cache,id=nexent-data-process-pip-${TARGETARCH},target=/root/.cache/pip,sharing=locked \ + pip install uv $(test -n "$MIRROR" && echo "-i $MIRROR") +WORKDIR /opt/backend +# Layer 1: install base dependencies +COPY backend/pyproject.toml /opt/backend/pyproject.toml +COPY sdk /opt/sdk +RUN --mount=type=cache,id=nexent-data-process-uv-${TARGETARCH},target=/root/.cache/uv,sharing=locked \ + printf '%s\n' \ + cuda-bindings \ + cuda-pathfinder \ + cuda-toolkit \ + nvidia-cublas \ + nvidia-cublas-cu11 \ + nvidia-cublas-cu12 \ + nvidia-cublas-cu13 \ + nvidia-cuda-cccl \ + nvidia-cuda-crt \ + nvidia-cuda-culibos \ + nvidia-cuda-cupti \ + nvidia-cuda-cupti-cu11 \ + nvidia-cuda-cupti-cu12 \ + nvidia-cuda-cupti-cu13 \ + nvidia-cuda-cuxxfilt \ + nvidia-cuda-nvcc \ + nvidia-cuda-nvrtc \ + nvidia-cuda-nvrtc-cu11 \ + nvidia-cuda-nvrtc-cu12 \ + nvidia-cuda-nvrtc-cu13 \ + nvidia-cuda-opencl \ + nvidia-cuda-profiler-api \ + nvidia-cuda-runtime \ + nvidia-cuda-runtime-cu11 \ + nvidia-cuda-runtime-cu12 \ + nvidia-cuda-runtime-cu13 \ + nvidia-cuda-sanitizer-api \ + nvidia-cudnn \ + nvidia-cudnn-cu11 \ + nvidia-cudnn-cu12 \ + nvidia-cudnn-cu13 \ + nvidia-cufft \ + nvidia-cufft-cu11 \ + nvidia-cufft-cu12 \ + nvidia-cufft-cu13 \ + nvidia-cufile \ + nvidia-cufile-cu11 \ + nvidia-cufile-cu12 \ + nvidia-cufile-cu13 \ + nvidia-curand \ + nvidia-curand-cu11 \ + nvidia-curand-cu12 \ + nvidia-curand-cu13 \ + nvidia-cusolver \ + nvidia-cusolver-cu11 \ + nvidia-cusolver-cu12 \ + nvidia-cusolver-cu13 \ + nvidia-cusparse \ + nvidia-cusparse-cu11 \ + nvidia-cusparse-cu12 \ + nvidia-cusparse-cu13 \ + nvidia-cusparselt \ + nvidia-cusparselt-cu12 \ + nvidia-cusparselt-cu13 \ + nvidia-nccl \ + nvidia-nccl-cu11 \ + nvidia-nccl-cu12 \ + nvidia-nccl-cu13 \ + nvidia-npp \ + nvidia-nvfatbin \ + nvidia-nvjitlink \ + nvidia-nvjitlink-cu11 \ + nvidia-nvjitlink-cu12 \ + nvidia-nvjitlink-cu13 \ + nvidia-nvjpeg \ + nvidia-nvml-dev \ + nvidia-nvptxcompiler \ + nvidia-nvshmem \ + nvidia-nvshmem-cu12 \ + nvidia-nvshmem-cu13 \ + nvidia-nvtx \ + nvidia-nvtx-cu11 \ + nvidia-nvtx-cu12 \ + nvidia-nvtx-cu13 \ + nvidia-nvvm \ + triton \ + > /tmp/nvidia-excludes.txt && \ + mirror_index_args="" && \ + if [ -n "$MIRROR" ]; then \ + mirror_index_args="--default-index ${MIRROR}"; \ + fi && \ + if [ "$DATA_PROCESS_DEPENDENCY_VARIANT" = "cpu" ]; then \ + torch_args="--torch-backend cpu --excludes /tmp/nvidia-excludes.txt"; \ + elif [ "$DATA_PROCESS_DEPENDENCY_VARIANT" = "gpu" ]; then \ + torch_args=""; \ + else \ + echo "Unsupported DATA_PROCESS_DEPENDENCY_VARIANT: ${DATA_PROCESS_DEPENDENCY_VARIANT}" >&2; \ + exit 1; \ + fi && \ + uv venv .venv && \ + uv pip install --python .venv/bin/python --link-mode copy $mirror_index_args $torch_args ".[data-process]" && \ + uv pip install --python .venv/bin/python --link-mode copy $mirror_index_args $torch_args "/opt/sdk[data-process]" && \ + if [ "$DATA_PROCESS_DEPENDENCY_VARIANT" = "cpu" ]; then \ + .venv/bin/python -c 'import importlib.metadata as metadata, importlib.util, sys; blocked = sorted(name for name in ((dist.metadata.get("Name") or "").lower() for dist in metadata.distributions()) if name == "triton" or name.startswith("nvidia-") or name.startswith("cuda-")); blocked and sys.exit("CPU data-process image must not install CUDA packages: " + ", ".join(blocked)); spec = importlib.util.find_spec("torch"); torch = __import__("torch") if spec else None; torch is not None and torch.cuda.is_available() and sys.exit("CPU data-process image unexpectedly reports CUDA availability"); print(f"Using CPU PyTorch {torch.__version__}") if torch else None'; \ + fi + +FROM data-process-base AS final +ARG TARGETARCH + +ENV VIRTUAL_ENV=/opt/backend/.venv +ENV PATH="$VIRTUAL_ENV/bin:/usr/bin:/bin:/usr/local/bin:$PATH" +WORKDIR /opt/backend + +RUN --mount=type=cache,id=nexent-data-process-apt-cache-${TARGETARCH},target=/var/cache/apt,sharing=locked \ + --mount=type=cache,id=nexent-data-process-apt-lists-${TARGETARCH},target=/var/lib/apt/lists,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + mkdir -p /var/cache/apt/archives /var/lib/apt/lists/partial && \ + apt-get update && \ + apt-get install -y --no-install-recommends --fix-missing \ + libreoffice \ + fontconfig \ + fonts-noto-cjk && \ + fc-cache -fv && \ + apt-get autoremove -y && \ + rm -rf /tmp/* /var/tmp/* + +RUN --mount=type=bind,source=model-assets,target=/tmp/model-assets,readonly \ + mkdir -p /opt/models && \ + cp -a /tmp/model-assets/clip-vit-base-patch32 /opt/models/clip-vit-base-patch32 && \ + cp -a /tmp/model-assets/nltk_data /opt/models/nltk_data && \ + cp -a /tmp/model-assets/table-transformer-structure-recognition /opt/models/table-transformer-structure-recognition && \ + cp -a /tmp/model-assets/yolox /opt/models/yolox + +COPY --from=data-process-deps /opt/backend/.venv /opt/backend/.venv +COPY --from=data-process-deps /opt/sdk /opt/sdk + +# Pre-download tiktoken cl100k_base model to avoid network issues during runtime. +RUN python -c "import tiktoken; enc = tiktoken.get_encoding('cl100k_base')" + +# Layer 3: copy backend code +COPY backend /opt/backend +COPY VERSION /opt/nexent/VERSION +COPY deploy/common/run-sql-migrations.sh deploy/common/start-backend.sh /opt/nexent/scripts/ +RUN chmod +x /opt/nexent/scripts/run-sql-migrations.sh /opt/nexent/scripts/start-backend.sh + +WORKDIR /opt + +# Expose the service port +EXPOSE 5012 diff --git a/deploy/images/dockerfiles/docs/Dockerfile b/deploy/images/dockerfiles/docs/Dockerfile new file mode 100644 index 000000000..f94c4351e --- /dev/null +++ b/deploy/images/dockerfiles/docs/Dockerfile @@ -0,0 +1,42 @@ +# syntax=docker/dockerfile:1.7 + +FROM node:20-alpine AS builder +ARG MIRROR +ARG TARGETARCH + +WORKDIR /app +COPY doc/package.json ./package.json + +RUN --mount=type=cache,id=nexent-docs-npm-${TARGETARCH},target=/root/.npm,sharing=locked \ + if [ -n "$MIRROR" ]; then npm config set registry "$MIRROR"; fi && \ + npm install --verbose + +COPY doc . + +RUN \ + npm run docs:build + +FROM nginx:1.27-alpine +ARG APK_MIRROR + +RUN if [ "$APK_MIRROR" = "tsinghua" ]; then \ + echo "https://mirrors.tuna.tsinghua.edu.cn/alpine/latest-stable/main" > /etc/apk/repositories && \ + echo "https://mirrors.tuna.tsinghua.edu.cn/alpine/latest-stable/community" >> /etc/apk/repositories; \ + fi && \ + printf '%s\n' \ + 'server {' \ + ' listen 4173;' \ + ' server_name _;' \ + ' root /usr/share/nginx/html;' \ + ' index index.html;' \ + ' location / {' \ + ' try_files $uri $uri/ /index.html;' \ + ' }' \ + '}' > /etc/nginx/conf.d/default.conf + +COPY --from=builder /app/docs/.vitepress/dist /usr/share/nginx/html + +EXPOSE 4173 + +HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ + CMD wget -q --spider http://localhost:4173/ || exit 1 diff --git a/deploy/images/dockerfiles/main/Dockerfile b/deploy/images/dockerfiles/main/Dockerfile new file mode 100644 index 000000000..2741e7f81 --- /dev/null +++ b/deploy/images/dockerfiles/main/Dockerfile @@ -0,0 +1,69 @@ +# syntax=docker/dockerfile:1.7 + +FROM python:3.11-slim AS base +ARG MIRROR +ARG APT_MIRROR +ARG TARGETARCH +LABEL authors="nexent" + +# Set correct permissions as root +USER root +RUN umask 0022 + +# Configure apt sources based on build argument +RUN --mount=type=cache,id=nexent-main-apt-cache-${TARGETARCH},target=/var/cache/apt,sharing=locked \ + --mount=type=cache,id=nexent-main-apt-lists-${TARGETARCH},target=/var/lib/apt/lists,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + mkdir -p /var/cache/apt/archives /var/lib/apt/lists/partial && \ + if [ "$APT_MIRROR" = "tsinghua" ]; then \ + rm -f /etc/apt/sources.list.d/* && \ + echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list && \ + echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \ + echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-backports main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \ + echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bookworm-security main contrib non-free non-free-firmware" >> /etc/apt/sources.list; \ + fi && \ + apt-get update && apt-get install -y --no-install-recommends curl postgresql-client + +FROM base AS builder +ARG MIRROR +ARG TARGETARCH + +RUN --mount=type=cache,id=nexent-main-pip-${TARGETARCH},target=/root/.cache/pip,sharing=locked \ + pip install uv $(test -n "$MIRROR" && echo "-i $MIRROR") +WORKDIR /opt/backend + +# Layer 0: install base dependencies +COPY backend/pyproject.toml /opt/backend/pyproject.toml +RUN --mount=type=cache,id=nexent-main-uv-${TARGETARCH},target=/root/.cache/uv,sharing=locked \ + uv sync --link-mode copy $(test -n "$MIRROR" && echo "-i $MIRROR") +# Layer 1: install sdk in link mode +COPY sdk /opt/sdk +RUN --mount=type=cache,id=nexent-main-uv-${TARGETARCH},target=/root/.cache/uv,sharing=locked \ + uv pip install --link-mode copy "/opt/sdk[performance]" $(test -n "$MIRROR" && echo "-i $MIRROR") + +FROM base AS final + +ENV VIRTUAL_ENV=/opt/backend/.venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +WORKDIR /opt/backend + +COPY --from=builder /opt/backend/.venv /opt/backend/.venv +COPY --from=builder /opt/sdk /opt/sdk + +# Pre-download tiktoken cl100k_base model to avoid network issues during runtime. +RUN python -c "import tiktoken; enc = tiktoken.get_encoding('cl100k_base')" + +# Layer 2: copy backend code +COPY backend /opt/backend +COPY VERSION /opt/nexent/VERSION +COPY deploy/common/run-sql-migrations.sh deploy/common/start-backend.sh /opt/nexent/scripts/ +RUN chmod +x /opt/nexent/scripts/run-sql-migrations.sh /opt/nexent/scripts/start-backend.sh + +# Create SSH key directory for Terminal tool +RUN mkdir -p /opt/ssh-keys +VOLUME ["/opt/ssh-keys"] + +WORKDIR /opt + +# Expose the service port +EXPOSE 5010 diff --git a/make/mcp/Dockerfile b/deploy/images/dockerfiles/mcp/Dockerfile similarity index 56% rename from make/mcp/Dockerfile rename to deploy/images/dockerfiles/mcp/Dockerfile index e011bf5fe..5f8fc1b44 100644 --- a/make/mcp/Dockerfile +++ b/deploy/images/dockerfiles/mcp/Dockerfile @@ -1,14 +1,21 @@ +# syntax=docker/dockerfile:1.7 + FROM python:3.11-slim ARG MIRROR ARG APT_MIRROR +ARG TARGETARCH # Set correct permissions as root USER root RUN umask 0022 # Configure apt sources based on build argument -RUN if [ "$APT_MIRROR" = "tsinghua" ]; then \ +RUN --mount=type=cache,id=nexent-mcp-apt-cache-${TARGETARCH},target=/var/cache/apt,sharing=locked \ + --mount=type=cache,id=nexent-mcp-apt-lists-${TARGETARCH},target=/var/lib/apt/lists,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + mkdir -p /var/cache/apt/archives /var/lib/apt/lists/partial && \ + if [ "$APT_MIRROR" = "tsinghua" ]; then \ rm -f /etc/apt/sources.list.d/* && \ echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm main contrib non-free non-free-firmware" > /etc/apt/sources.list && \ echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bookworm-updates main contrib non-free non-free-firmware" >> /etc/apt/sources.list && \ @@ -16,36 +23,36 @@ RUN if [ "$APT_MIRROR" = "tsinghua" ]; then \ echo "deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bookworm-security main contrib non-free non-free-firmware" >> /etc/apt/sources.list; \ fi && \ apt-get update && \ - apt-get install -y --no-install-recommends curl ca-certificates gnupg xz-utils && \ - rm -rf /var/lib/apt/lists/* + apt-get install -y --no-install-recommends curl ca-certificates gnupg xz-utils + +# Install Node.js 20 from official binaries (pin exact version to avoid repo issues) +ARG NODE_VERSION=20.17.0 +RUN --mount=type=cache,id=nexent-mcp-nodejs-${TARGETARCH},target=/var/cache/nodejs,sharing=locked \ + set -eu && \ + arch="$(dpkg --print-architecture)" && \ + case "${arch}" in \ + amd64) node_arch="x64" ;; \ + arm64) node_arch="arm64" ;; \ + *) echo "Unsupported architecture: ${arch}" >&2; exit 1 ;; \ + esac && \ + node_tarball="/var/cache/nodejs/node-v${NODE_VERSION}-linux-${node_arch}.tar.xz" && \ + if [ ! -f "$node_tarball" ]; then \ + curl -fsSLo "$node_tarball" "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${node_arch}.tar.xz"; \ + fi && \ + tar -C /usr/local --strip-components=1 -xJf "$node_tarball" && \ + node -v && npm -v # Optional pip mirror for Python packages RUN if [ -n "$MIRROR" ]; then pip config set global.index-url "$MIRROR"; fi -# Install uv (fast Python package installer) -RUN pip install --no-cache-dir uv - ARG MCP_PROXY_VERSION WORKDIR /opt # Install mcp-proxy from PyPI (optionally pinned) -RUN if [ -n "$MCP_PROXY_VERSION" ]; then \ - pip install --no-cache-dir "mcp-proxy==$MCP_PROXY_VERSION"; \ +RUN --mount=type=cache,id=nexent-mcp-pip-${TARGETARCH},target=/root/.cache/pip,sharing=locked \ + if [ -n "$MCP_PROXY_VERSION" ]; then \ + pip install "mcp-proxy==$MCP_PROXY_VERSION"; \ else \ - pip install --no-cache-dir mcp-proxy; \ + pip install mcp-proxy; \ fi - -# Install Node.js 20 from official binaries (pin exact version to avoid repo issues) -ARG NODE_VERSION=20.17.0 -RUN set -eu && \ - arch="$(dpkg --print-architecture)" && \ - case "${arch}" in \ - amd64) node_arch="x64" ;; \ - arm64) node_arch="arm64" ;; \ - *) echo "Unsupported architecture: ${arch}" >&2; exit 1 ;; \ - esac && \ - curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${node_arch}.tar.xz" && \ - tar -C /usr/local --strip-components=1 -xJf "node-v${NODE_VERSION}-linux-${node_arch}.tar.xz" && \ - rm "node-v${NODE_VERSION}-linux-${node_arch}.tar.xz" && \ - node -v && npm -v \ No newline at end of file diff --git a/deploy/images/dockerfiles/terminal/Dockerfile b/deploy/images/dockerfiles/terminal/Dockerfile new file mode 100644 index 000000000..46f12058e --- /dev/null +++ b/deploy/images/dockerfiles/terminal/Dockerfile @@ -0,0 +1,65 @@ +# syntax=docker/dockerfile:1.7 + +FROM ubuntu:24.04 + +ARG TERMINAL_VARIANT=slim +ARG TARGETARCH + +ENV CONDA_DIR=/opt/conda + +RUN --mount=type=cache,id=nexent-terminal-apt-cache-${TARGETARCH},target=/var/cache/apt,sharing=locked \ + --mount=type=cache,id=nexent-terminal-apt-lists-${TARGETARCH},target=/var/lib/apt/lists,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + mkdir -p /var/cache/apt/archives /var/lib/apt/lists/partial && \ + if [ "$TERMINAL_VARIANT" != "slim" ] && [ "$TERMINAL_VARIANT" != "conda" ]; then \ + echo "Unsupported TERMINAL_VARIANT: ${TERMINAL_VARIANT}" >&2; \ + exit 1; \ + fi && \ + apt-get update --fix-missing && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + openssh-server \ + curl \ + wget \ + git \ + python3 \ + python3-pip \ + python3-venv && \ + if [ "$TERMINAL_VARIANT" = "conda" ]; then \ + apt-get install -y --no-install-recommends vim build-essential; \ + fi + +# Configure SSH - enable root login + enable password authentication. +RUN mkdir /var/run/sshd && \ + sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \ + sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config + +RUN --mount=type=cache,id=nexent-terminal-miniconda-${TARGETARCH},target=/var/cache/miniconda,sharing=locked \ + if [ "$TERMINAL_VARIANT" = "conda" ]; then \ + arch="${TARGETARCH:-$(dpkg --print-architecture)}" && \ + case "$arch" in \ + amd64|x86_64) conda_arch="x86_64" ;; \ + arm64|aarch64) conda_arch="aarch64" ;; \ + *) echo "Unsupported architecture: ${arch}" >&2; exit 1 ;; \ + esac && \ + miniconda_installer="/var/cache/miniconda/Miniconda3-latest-Linux-${conda_arch}.sh" && \ + if [ ! -f "$miniconda_installer" ]; then \ + wget "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${conda_arch}.sh" -O "$miniconda_installer"; \ + fi && \ + bash "$miniconda_installer" -b -p "$CONDA_DIR" && \ + "$CONDA_DIR/bin/conda" init; \ + else \ + mkdir -p "$CONDA_DIR"; \ + fi + +ENV PATH="$CONDA_DIR/bin:$PATH" + +RUN mkdir -p /root/.ssh /opt/terminal && \ + chmod 700 /root/.ssh + +WORKDIR /opt + +COPY --chmod=755 deploy/images/dockerfiles/terminal/entrypoint.sh /entrypoint.sh + +EXPOSE 22 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/make/terminal/entrypoint.sh b/deploy/images/dockerfiles/terminal/entrypoint.sh similarity index 100% rename from make/terminal/entrypoint.sh rename to deploy/images/dockerfiles/terminal/entrypoint.sh diff --git a/deploy/images/dockerfiles/web/Dockerfile b/deploy/images/dockerfiles/web/Dockerfile new file mode 100644 index 000000000..fb1a145ee --- /dev/null +++ b/deploy/images/dockerfiles/web/Dockerfile @@ -0,0 +1,72 @@ +# syntax=docker/dockerfile:1.7 + +# Build stage +FROM node:20-alpine AS builder +ARG MIRROR +ARG TARGETARCH + +# Build Next.js application +WORKDIR /opt/frontend +COPY frontend/package.json ./package.json + +# Use BuildKit named cache for npm downloads across builds. +RUN --mount=type=cache,id=nexent-web-npm-${TARGETARCH},target=/root/.npm,sharing=locked \ + if [ -n "$MIRROR" ]; then npm config set registry "$MIRROR"; fi && \ + npm install --verbose + +COPY frontend /opt/frontend + +RUN --mount=type=cache,id=nexent-web-next-${TARGETARCH},target=/opt/frontend/.next/cache,sharing=locked \ + NODE_ENV=production npm run build && \ + mkdir -p ../frontend-dist && \ + cp -r .next/standalone/. ../frontend-dist/ && \ + mkdir -p ../frontend-dist/.next && \ + cp -r .next/static ../frontend-dist/.next/static && \ + cp -r public ../frontend-dist/ && \ + cp server.js ../frontend-dist/server.js && \ + mkdir -p ../frontend-dist/node_modules/next/dist/compiled && \ + cp -r node_modules/next/dist/compiled/. ../frontend-dist/node_modules/next/dist/compiled/ && \ + mkdir -p ../frontend-dist/node_modules && \ + cp -r \ + node_modules/cookie \ + node_modules/dotenv \ + node_modules/eventemitter3 \ + node_modules/follow-redirects \ + node_modules/http-proxy \ + node_modules/requires-port \ + ../frontend-dist/node_modules/ && \ + rm -rf ../frontend-dist/.next/cache + +# Production stage +FROM node:20-alpine +ARG APK_MIRROR +ARG TARGETARCH +LABEL authors="nexent" + +# Configure Alpine mirrors if specified +RUN if [ "$APK_MIRROR" = "tsinghua" ]; then \ + echo "https://mirrors.tuna.tsinghua.edu.cn/alpine/latest-stable/main" > /etc/apk/repositories && \ + echo "https://mirrors.tuna.tsinghua.edu.cn/alpine/latest-stable/community" >> /etc/apk/repositories; \ + fi + +# Update package index, upgrade busybox first, then install curl +# This avoids trigger script issues in cross-platform builds with QEMU emulation +RUN --mount=type=cache,id=nexent-web-apk-${TARGETARCH},target=/var/cache/apk,sharing=locked \ + mkdir -p /var/cache/apk && \ + apk update && \ + (apk upgrade busybox || true) && \ + apk add --no-scripts curl + +WORKDIR /opt/frontend-dist + +# Copy only the necessary files from builder +COPY --from=builder /opt/frontend-dist . + +ENV NODE_ENV=production +ENV HOSTNAME=localhost + +# Expose the service port +EXPOSE 3000 + +# Start the server +CMD ["node", "server.js"] diff --git a/k8s/helm/create-suadmin.sh b/deploy/k8s/create-suadmin.sh similarity index 95% rename from k8s/helm/create-suadmin.sh rename to deploy/k8s/create-suadmin.sh index 245734f4e..476fe7f91 100644 --- a/k8s/helm/create-suadmin.sh +++ b/deploy/k8s/create-suadmin.sh @@ -6,11 +6,21 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" CHART_DIR="$SCRIPT_DIR/nexent" COMMON_VALUES="$CHART_DIR/charts/nexent-common/values.yaml" NAMESPACE="nexent" RELEASE_NAME="nexent" SUPER_ADMIN_EMAIL="suadmin@nexent.com" +DEPLOYMENT_COMMON="$DEPLOY_ROOT/common/common.sh" + +if [ -f "$DEPLOYMENT_COMMON" ]; then + # shellcheck source=/dev/null + source "$DEPLOYMENT_COMMON" +else + echo "Error: shared deployment helper not found: $DEPLOYMENT_COMMON" + exit 1 +fi # Prompt user to enter password for super admin user with confirmation prompt_super_admin_password() { @@ -22,6 +32,7 @@ prompt_super_admin_password() { echo "" >&2 echo "🔐 Super Admin User Password Setup" >&2 echo " Email: suadmin@nexent.com" >&2 + echo " Requirement: $(deployment_password_validation_message)" >&2 echo "" >&2 while [ $attempts -lt $max_attempts ]; do @@ -35,6 +46,12 @@ prompt_super_admin_password() { continue fi + if ! deployment_validate_password "$password"; then + echo " ❌ $(deployment_password_validation_message)" >&2 + attempts=$((attempts + 1)) + continue + fi + echo " 🔐 Please confirm the password:" >&2 read -s password_confirm echo "" >&2 diff --git a/deploy/k8s/deploy.sh b/deploy/k8s/deploy.sh new file mode 100755 index 000000000..1e727dec2 --- /dev/null +++ b/deploy/k8s/deploy.sh @@ -0,0 +1,1183 @@ +#!/bin/bash +# Helm Deployment Script for Nexent +# Usage: ./deploy.sh [apply] [options] +# +# Deploy only. Use uninstall.sh for uninstall and cleanup commands. + +set -e + +# Use absolute path relative to the script location +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +CHART_DIR="$SCRIPT_DIR/helm/nexent" +COMMON_VALUES="$CHART_DIR/charts/nexent-common/values.yaml" +NAMESPACE="nexent" +RELEASE_NAME="nexent" +DEPLOYMENT_COMMON="$DEPLOY_ROOT/common/common.sh" +VERSION_HELPER="$DEPLOY_ROOT/common/version.sh" + +# Constants for deployment options +K8S_ROOT="$SCRIPT_DIR" +CONST_FILE="$PROJECT_ROOT/backend/consts/const.py" +DEPLOY_OPTIONS_FILE="$SCRIPT_DIR/deploy.options" +GENERATED_VALUES="$CHART_DIR/generated-values.yaml" +GENERATED_RUNTIME_VALUES="$CHART_DIR/generated-runtime-values.yaml" +GENERATED_SECRETS_VALUES="$CHART_DIR/generated-secrets-values.yaml" +GENERATED_PERSISTENCE_VALUES="$CHART_DIR/generated-persistence-values.yaml" +ROOT_ENV_FILE="$PROJECT_ROOT/.env" +SQL_INIT_FILE="$DEPLOY_ROOT/sql/init.sql" +SUPABASE_SQL_DIR="$DEPLOY_ROOT/sql/supabase" + +if [ -f "$DEPLOYMENT_COMMON" ]; then + # shellcheck source=/dev/null + source "$DEPLOYMENT_COMMON" +else + echo "Error: shared deployment helper not found: $DEPLOYMENT_COMMON" + exit 1 +fi + +if [ -f "$VERSION_HELPER" ]; then + # shellcheck source=/dev/null + source "$VERSION_HELPER" +fi + +# Global variables for deployment options +IS_MAINLAND="" +APP_VERSION="" +DEPLOYMENT_VERSION="" +VERSION_CHOICE_SAVED="" +PERSISTENCE_MODE="local" +STORAGE_CLASS_NAME="" +LOCAL_PATH="/var/lib/nexent-data" +LOCAL_NODE_NAME="" +EXISTING_CLAIM_PREFIX="" +K8S_WAIT_TIMEOUT_SECONDS="${NEXENT_K8S_WAIT_TIMEOUT_SECONDS:-600}" + +# Parse command line arguments. The optional "apply" command is kept as a deploy alias. +COMMAND="apply" +case "${1:-}" in + --help|-h) + COMMAND="help" + shift + ;; + ""|--*) + ;; + apply|deploy) + COMMAND="apply" + shift + ;; + delete|delete-all|clean) + echo "K8s uninstall and cleanup have moved to uninstall.sh." + echo "Use: bash uninstall.sh ${1}" + exit 1 + ;; + *) + echo "Unknown command: $1" + echo "Usage: $0 [apply] [options]" + echo "Uninstall: bash uninstall.sh" + exit 1 + ;; +esac +if [ "$COMMAND" = "apply" ] && { [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ]; }; then + COMMAND="help" + shift +fi +ORIGINAL_ARGS=("$@") + +while [[ $# -gt 0 ]]; do + case "$1" in + --is-mainland) + IS_MAINLAND="$2" + shift 2 + ;; + --version) + APP_VERSION="$2" + shift 2 + ;; + --deployment-version) + DEPLOYMENT_VERSION="$2" + shift 2 + ;; + --persistence-mode) + PERSISTENCE_MODE="$2" + shift 2 + ;; + --storage-class|--storageclass|--storage-class-name|--sc) + STORAGE_CLASS_NAME="$2" + shift 2 + ;; + --local-path) + LOCAL_PATH="$2" + shift 2 + ;; + --local-node-name) + LOCAL_NODE_NAME="$2" + shift 2 + ;; + --existing-claim-prefix) + EXISTING_CLAIM_PREFIX="$2" + shift 2 + ;; + --wait-timeout) + K8S_WAIT_TIMEOUT_SECONDS="$2" + shift 2 + ;; + --rotate-secrets|--refresh-es-key) + shift + ;; + *) + shift + ;; + esac +done + +cd "$SCRIPT_DIR" +deployment_source_root_env "$PROJECT_ROOT" "$PROJECT_ROOT/docker" || exit 1 + +# Helper function to sanitize input (remove Windows CR) +sanitize_input() { + local input="$1" + printf "%s" "$input" | tr -d '\r' +} + +apply_deployment_common_config() { + if [ -z "$APP_VERSION" ]; then + APP_VERSION=$(get_app_version) + fi + if [ -n "$APP_VERSION" ]; then + export APP_VERSION + fi + + deployment_prepare_config "${ORIGINAL_ARGS[@]}" || return 1 + + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "supabase"; then + DEPLOYMENT_VERSION="full" + else + DEPLOYMENT_VERSION="speed" + fi + + APP_VERSION="$DEPLOYMENT_APP_VERSION" + VERSION_CHOICE_SAVED="$DEPLOYMENT_VERSION" + + case "$DEPLOYMENT_REGISTRY_PROFILE" in + mainland) + IS_MAINLAND_SAVED="Y" + source "$DEPLOY_ROOT/env/image-source.mainland.env" + ;; + general|local-latest) + IS_MAINLAND_SAVED="N" + source "$DEPLOY_ROOT/env/image-source.general.env" + ;; + esac + + deployment_apply_image_source + deployment_render_helm_values "$GENERATED_VALUES" + render_k8s_runtime_config_values "$GENERATED_RUNTIME_VALUES" + render_persistence_values "$GENERATED_PERSISTENCE_VALUES" + deployment_print_summary k8s +} + + +persistence_existing_claim() { + local component="$1" + if [ -n "$EXISTING_CLAIM_PREFIX" ]; then + printf '%s-%s' "$EXISTING_CLAIM_PREFIX" "$component" + fi +} + +render_one_persistence_values() { + local output_file="$1" + local chart="$2" + local component="$3" + local size="$4" + local storage_class="$STORAGE_CLASS_NAME" + [ -n "$storage_class" ] || storage_class="nexent-local" + [ "$PERSISTENCE_MODE" = "dynamic" ] && [ "$STORAGE_CLASS_NAME" = "" ] && storage_class="" + + { + printf '%s:\n' "$chart" + printf ' persistence:\n' + printf ' mode: "%s"\n' "$PERSISTENCE_MODE" + printf ' storageClassName: "%s"\n' "$storage_class" + printf ' accessModes:\n' + printf ' - ReadWriteOnce\n' + printf ' localPath: "%s/%s"\n' "$LOCAL_PATH" "$component" + printf ' existingClaim: "%s"\n' "$(persistence_existing_claim "$component")" + printf ' storage:\n' + printf ' size: "%s"\n' "$size" + } >> "$output_file" +} + +render_monitoring_persistence_values() { + local output_file="$1" + local storage_class="$STORAGE_CLASS_NAME" + [ -n "$storage_class" ] || storage_class="nexent-local" + [ "$PERSISTENCE_MODE" = "dynamic" ] && [ "$STORAGE_CLASS_NAME" = "" ] && storage_class="" + + { + printf 'nexent-monitoring:\n' + printf ' persistence:\n' + printf ' enabled: true\n' + printf ' mode: "%s"\n' "$PERSISTENCE_MODE" + printf ' storageClassName: "%s"\n' "$storage_class" + printf ' accessModes:\n' + printf ' - ReadWriteOnce\n' + printf ' localPath: "%s"\n' "$LOCAL_PATH" + printf ' existingClaimPrefix: "%s"\n' "$EXISTING_CLAIM_PREFIX" + } >> "$output_file" +} + +render_shared_storage_persistence_values() { + local output_file="$1" + local storage_class="$STORAGE_CLASS_NAME" + [ -n "$storage_class" ] || storage_class="nexent-local" + [ "$PERSISTENCE_MODE" = "dynamic" ] && [ "$STORAGE_CLASS_NAME" = "" ] && storage_class="" + + { + printf 'global:\n' + printf ' sharedStorage:\n' + printf ' mode: "%s"\n' "$PERSISTENCE_MODE" + printf ' storageClassName: "%s"\n' "$storage_class" + printf ' accessModes:\n' + printf ' - ReadWriteOnce\n' + printf ' workspace:\n' + printf ' size: "10Gi"\n' + printf ' localPath: "/var/lib/nexent"\n' + printf ' existingClaim: "%s"\n' "$(persistence_existing_claim "nexent-workspace")" + printf ' skills:\n' + printf ' size: "5Gi"\n' + printf ' localPath: "%s/skills"\n' "$LOCAL_PATH" + printf ' existingClaim: "%s"\n' "$(persistence_existing_claim "nexent-skills")" + } >> "$output_file" +} + +render_persistence_values() { + local output_file="$1" + case "$PERSISTENCE_MODE" in + local|dynamic|existing) ;; + *) + echo "Unsupported persistence mode: $PERSISTENCE_MODE" + echo "Use local, dynamic, or existing." + exit 1 + ;; + esac + + { + echo "# Generated persistence overrides" + } > "$output_file" + + render_shared_storage_persistence_values "$output_file" + render_one_persistence_values "$output_file" "nexent-elasticsearch" "nexent-elasticsearch" "20Gi" + render_one_persistence_values "$output_file" "nexent-postgresql" "nexent-postgresql" "10Gi" + render_one_persistence_values "$output_file" "nexent-redis" "nexent-redis" "5Gi" + render_one_persistence_values "$output_file" "nexent-minio" "nexent-minio" "20Gi" + render_one_persistence_values "$output_file" "nexent-supabase-db" "nexent-supabase-db" "10Gi" + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring"; then + render_monitoring_persistence_values "$output_file" + fi +} + +yaml_quote() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + printf '"%s"' "$value" +} + +env_or_default() { + local key="$1" + local default_value="$2" + if [ "${!key+x}" = "x" ]; then + printf '%s' "${!key}" + else + printf '%s' "$default_value" + fi +} + +render_yaml_literal_file() { + local key="$1" + local file="$2" + local key_indent="$3" + local content_indent="$4" + local key_padding + local content_padding + + if [ ! -f "$file" ]; then + echo "Error: SQL file not found: $file" + exit 1 + fi + + key_padding="$(printf '%*s' "$key_indent" '')" + content_padding="$(printf '%*s' "$content_indent" '')" + printf '%s%s: |\n' "$key_padding" "$key" + sed "s/^/${content_padding}/" "$file" + printf '\n' +} + +sql_files_checksum() { + local payload="" + local file rel checksum + if [ -f "$SQL_INIT_FILE" ]; then + checksum="$(deployment_sha256_file "$SQL_INIT_FILE")" + payload="${payload}init.sql:${checksum}"$'\n' + fi + if [ -d "$DEPLOY_ROOT/sql/migrations" ]; then + while IFS= read -r file; do + [ -n "$file" ] || continue + rel="${file#"$DEPLOY_ROOT/sql/"}" + checksum="$(deployment_sha256_file "$file")" + payload="${payload}${rel}:${checksum}"$'\n' + done < <(find "$DEPLOY_ROOT/sql/migrations" -maxdepth 1 -type f -name '*.sql' -print | sort -V) + fi + if [ -d "$SUPABASE_SQL_DIR" ]; then + while IFS= read -r file; do + [ -n "$file" ] || continue + rel="${file#"$DEPLOY_ROOT/sql/"}" + checksum="$(deployment_sha256_file "$file")" + payload="${payload}${rel}:${checksum}"$'\n' + done < <(find "$SUPABASE_SQL_DIR" -maxdepth 1 -type f -name '*.sql' -print | sort -V) + fi + deployment_sha256_string "$payload" +} + +render_k8s_runtime_config_values() { + local output_file="$1" + local file + if [ ! -f "$SQL_INIT_FILE" ]; then + echo "Error: SQL init file not found: $SQL_INIT_FILE" + exit 1 + fi + if [ ! -d "$DEPLOY_ROOT/sql/migrations" ]; then + echo "Error: SQL migrations directory not found: $DEPLOY_ROOT/sql/migrations" + exit 1 + fi + if [ ! -d "$SUPABASE_SQL_DIR" ]; then + echo "Error: Supabase SQL directory not found: $SUPABASE_SQL_DIR" + exit 1 + fi + { + echo "global:" + echo " sqlFileNames:" + echo " migrations:" + while IFS= read -r file; do + [ -n "$file" ] || continue + printf ' - %s\n' "$(yaml_quote "$(basename "$file")")" + done < <(find "$DEPLOY_ROOT/sql/migrations" -maxdepth 1 -type f -name '*.sql' -print | sort -V) + echo " supabase:" + while IFS= read -r file; do + [ -n "$file" ] || continue + printf ' - %s\n' "$(yaml_quote "$(basename "$file")")" + done < <(find "$SUPABASE_SQL_DIR" -maxdepth 1 -type f -name '*.sql' -print | sort -V) + echo "nexent-common:" + echo " sqlFiles:" + render_yaml_literal_file "init" "$SQL_INIT_FILE" 4 6 + echo " migrations:" + while IFS= read -r file; do + [ -n "$file" ] || continue + render_yaml_literal_file "$(basename "$file")" "$file" 6 8 + done < <(find "$DEPLOY_ROOT/sql/migrations" -maxdepth 1 -type f -name '*.sql' -print | sort -V) + echo " supabase:" + while IFS= read -r file; do + [ -n "$file" ] || continue + render_yaml_literal_file "$(basename "$file")" "$file" 6 8 + done < <(find "$SUPABASE_SQL_DIR" -maxdepth 1 -type f -name '*.sql' -print | sort -V) + echo " config:" + echo " services:" + printf ' configUrl: %s\n' "$(yaml_quote "$(env_or_default CONFIG_SERVICE_URL "http://nexent-config:5010")")" + printf ' elasticsearchService: %s\n' "$(yaml_quote "$(env_or_default ELASTICSEARCH_SERVICE "http://nexent-config:5010/api")")" + printf ' runtimeUrl: %s\n' "$(yaml_quote "$(env_or_default RUNTIME_SERVICE_URL "http://nexent-runtime:5014")")" + printf ' mcpServer: %s\n' "$(yaml_quote "$(env_or_default NEXENT_MCP_SERVER "http://nexent-mcp:5011")")" + printf ' mcpManagementServer: %s\n' "$(yaml_quote "$(env_or_default MCP_MANAGEMENT_API "http://nexent-mcp:5015")")" + printf ' dataProcessService: %s\n' "$(yaml_quote "$(env_or_default DATA_PROCESS_SERVICE "http://nexent-data-process:5012/api")")" + printf ' northboundServer: %s\n' "$(yaml_quote "$(env_or_default NORTHBOUND_API_SERVER "http://nexent-northbound:5013/api")")" + printf ' northboundExternalUrl: %s\n' "$(yaml_quote "$(env_or_default NORTHBOUND_EXTERNAL_URL "")")" + echo " postgres:" + printf ' host: %s\n' "$(yaml_quote "$(env_or_default POSTGRES_HOST "nexent-postgresql")")" + printf ' user: %s\n' "$(yaml_quote "$(env_or_default POSTGRES_USER "root")")" + printf ' db: %s\n' "$(yaml_quote "$(env_or_default POSTGRES_DB "nexent")")" + printf ' port: %s\n' "$(yaml_quote "$(env_or_default POSTGRES_PORT "5432")")" + echo " redis:" + printf ' url: %s\n' "$(yaml_quote "$(env_or_default REDIS_URL "redis://nexent-redis:6379/0")")" + printf ' backendUrl: %s\n' "$(yaml_quote "$(env_or_default REDIS_BACKEND_URL "redis://nexent-redis:6379/1")")" + printf ' port: %s\n' "$(yaml_quote "$(env_or_default REDIS_PORT "6379")")" + echo " minio:" + printf ' endpoint: %s\n' "$(yaml_quote "$(env_or_default MINIO_ENDPOINT "http://nexent-minio:9000")")" + printf ' region: %s\n' "$(yaml_quote "$(env_or_default MINIO_REGION "cn-north-1")")" + printf ' defaultBucket: %s\n' "$(yaml_quote "$(env_or_default MINIO_DEFAULT_BUCKET "nexent")")" + echo " elasticsearch:" + printf ' host: %s\n' "$(yaml_quote "$(env_or_default ELASTICSEARCH_HOST "http://nexent-elasticsearch:9200")")" + printf ' javaOpts: %s\n' "$(yaml_quote "$(env_or_default ES_JAVA_OPTS "-Xms2g -Xmx2g")")" + printf ' diskWatermarkLow: %s\n' "$(yaml_quote "$(env_or_default ES_DISK_WATERMARK_LOW "85%")")" + printf ' diskWatermarkHigh: %s\n' "$(yaml_quote "$(env_or_default ES_DISK_WATERMARK_HIGH "90%")")" + printf ' diskWatermarkFloodStage: %s\n' "$(yaml_quote "$(env_or_default ES_DISK_WATERMARK_FLOOD_STAGE "95%")")" + printf ' skipProxy: %s\n' "$(yaml_quote "$(env_or_default skip_proxy "true")")" + printf ' umask: %s\n' "$(yaml_quote "$(env_or_default UMASK "0022")")" + printf ' skillsPath: %s\n' "$(yaml_quote "$(env_or_default SKILLS_PATH "/mnt/nexent-data/skills")")" + printf ' marketBackend: %s\n' "$(yaml_quote "$(env_or_default MARKET_BACKEND "http://60.204.251.153:8010")")" + echo " modelEngine:" + printf ' enabled: %s\n' "$(yaml_quote "$(env_or_default MODEL_ENGINE_ENABLED "false")")" + echo " voiceService:" + printf ' appid: %s\n' "$(yaml_quote "$(env_or_default APPID "app_id")")" + printf ' token: %s\n' "$(yaml_quote "$(env_or_default TOKEN "token")")" + printf ' cluster: %s\n' "$(yaml_quote "$(env_or_default CLUSTER "volcano_tts")")" + printf ' voiceType: %s\n' "$(yaml_quote "$(env_or_default VOICE_TYPE "zh_male_jieshuonansheng_mars_bigtts")")" + printf ' speedRatio: %s\n' "$(yaml_quote "$(env_or_default SPEED_RATIO "1.3")")" + echo " modelPath:" + printf ' clipModelPath: %s\n' "$(yaml_quote "$(env_or_default CLIP_MODEL_PATH "/opt/models/clip-vit-base-patch32")")" + printf ' nltkData: %s\n' "$(yaml_quote "$(env_or_default NLTK_DATA "/opt/models/nltk_data")")" + printf ' tableTransformerModelPath: %s\n' "$(yaml_quote "$(env_or_default TABLE_TRANSFORMER_MODEL_PATH "/opt/models/table-transformer-structure-recognition")")" + printf ' unstructuredDefaultModelInitializeParamsJsonPath: %s\n' "$(yaml_quote "$(env_or_default UNSTRUCTURED_DEFAULT_MODEL_INITIALIZE_PARAMS_JSON_PATH "/opt/models/yolox")")" + echo " terminal:" + printf ' sshPrivateKeyPath: %s\n' "$(yaml_quote "$(env_or_default SSH_PRIVATE_KEY_PATH "/path/to/openssh-server/ssh-keys/openssh_server_key")")" + echo " supabase:" + printf ' dashboardUsername: %s\n' "$(yaml_quote "$(env_or_default DASHBOARD_USERNAME "supabase")")" + printf ' dashboardPassword: %s\n' "$(yaml_quote "$(env_or_default DASHBOARD_PASSWORD "Huawei123")")" + printf ' siteUrl: %s\n' "$(yaml_quote "$(env_or_default SITE_URL "http://localhost:3011")")" + printf ' supabaseUrl: %s\n' "$(yaml_quote "$(env_or_default SUPABASE_URL "http://nexent-supabase-kong:8000")")" + printf ' apiExternalUrl: %s\n' "$(yaml_quote "$(env_or_default API_EXTERNAL_URL "http://nexent-supabase-kong:8000")")" + printf ' disableSignup: %s\n' "$(yaml_quote "$(env_or_default DISABLE_SIGNUP "false")")" + printf ' jwtExpiry: %s\n' "$(yaml_quote "$(env_or_default JWT_EXPIRY "3600")")" + printf ' debugJwtExpireSeconds: %s\n' "$(yaml_quote "$(env_or_default DEBUG_JWT_EXPIRE_SECONDS "0")")" + printf ' enableEmailSignup: %s\n' "$(yaml_quote "$(env_or_default ENABLE_EMAIL_SIGNUP "true")")" + printf ' enableEmailAutoconfirm: %s\n' "$(yaml_quote "$(env_or_default ENABLE_EMAIL_AUTOCONFIRM "true")")" + printf ' enableAnonymousUsers: %s\n' "$(yaml_quote "$(env_or_default ENABLE_ANONYMOUS_USERS "false")")" + printf ' enablePhoneSignup: %s\n' "$(yaml_quote "$(env_or_default ENABLE_PHONE_SIGNUP "false")")" + printf ' enablePhoneAutoconfirm: %s\n' "$(yaml_quote "$(env_or_default ENABLE_PHONE_AUTOCONFIRM "false")")" + printf ' inviteCode: %s\n' "$(yaml_quote "$(env_or_default INVITE_CODE "nexent2025")")" + printf ' mailerUrlpathsConfirmation: %s\n' "$(yaml_quote "$(env_or_default MAILER_URLPATHS_CONFIRMATION "/auth/v1/verify")")" + printf ' mailerUrlpathsInvite: %s\n' "$(yaml_quote "$(env_or_default MAILER_URLPATHS_INVITE "/auth/v1/verify")")" + printf ' mailerUrlpathsRecovery: %s\n' "$(yaml_quote "$(env_or_default MAILER_URLPATHS_RECOVERY "/auth/v1/verify")")" + printf ' mailerUrlpathsEmailChange: %s\n' "$(yaml_quote "$(env_or_default MAILER_URLPATHS_EMAIL_CHANGE "/auth/v1/verify")")" + printf ' postgresHost: %s\n' "$(yaml_quote "$(env_or_default SUPABASE_POSTGRES_HOST "nexent-supabase-db")")" + printf ' postgresDb: %s\n' "$(yaml_quote "$(env_or_default SUPABASE_POSTGRES_DB "supabase")")" + printf ' postgresPort: %s\n' "$(yaml_quote "$(env_or_default SUPABASE_POSTGRES_PORT "5436")")" + printf ' additionalRedirectUrls: %s\n' "$(yaml_quote "$(env_or_default ADDITIONAL_REDIRECT_URLS "")")" + echo " dataProcess:" + printf ' flowerPort: %s\n' "$(yaml_quote "$(env_or_default FLOWER_PORT "5555")")" + printf ' rayDashboardPort: %s\n' "$(yaml_quote "$(env_or_default RAY_DASHBOARD_PORT "8265")")" + printf ' rayDashboardHost: %s\n' "$(yaml_quote "$(env_or_default RAY_DASHBOARD_HOST "0.0.0.0")")" + printf ' rayActorNumCpus: %s\n' "$(yaml_quote "$(env_or_default RAY_ACTOR_NUM_CPUS "2")")" + printf ' rayNumCpus: %s\n' "$(yaml_quote "$(env_or_default RAY_NUM_CPUS "4")")" + printf ' rayObjectStoreMemoryGb: %s\n' "$(yaml_quote "$(env_or_default RAY_OBJECT_STORE_MEMORY_GB "0.25")")" + printf ' rayTempDir: %s\n' "$(yaml_quote "$(env_or_default RAY_TEMP_DIR "/tmp/ray")")" + printf ' rayLogLevel: %s\n' "$(yaml_quote "$(env_or_default RAY_LOG_LEVEL "INFO")")" + printf ' disableRayDashboard: %s\n' "$(yaml_quote "$(env_or_default DISABLE_RAY_DASHBOARD "true")")" + printf ' disableCeleryFlower: %s\n' "$(yaml_quote "$(env_or_default DISABLE_CELERY_FLOWER "true")")" + printf ' dockerEnvironment: %s\n' "$(yaml_quote "$(env_or_default DOCKER_ENVIRONMENT "false")")" + printf ' enableUploadImage: %s\n' "$(yaml_quote "$(env_or_default ENABLE_UPLOAD_IMAGE "false")")" + printf ' celeryWorkerPrefetchMultiplier: %s\n' "$(yaml_quote "$(env_or_default CELERY_WORKER_PREFETCH_MULTIPLIER "1")")" + printf ' celeryTaskTimeLimit: %s\n' "$(yaml_quote "$(env_or_default CELERY_TASK_TIME_LIMIT "3600")")" + printf ' elasticsearchRequestTimeout: %s\n' "$(yaml_quote "$(env_or_default ELASTICSEARCH_REQUEST_TIMEOUT "30")")" + printf ' queues: %s\n' "$(yaml_quote "$(env_or_default QUEUES "process_q,forward_q")")" + printf ' workerName: %s\n' "$(yaml_quote "$(env_or_default WORKER_NAME "")")" + printf ' workerConcurrency: %s\n' "$(yaml_quote "$(env_or_default WORKER_CONCURRENCY "4")")" + echo " telemetry:" + printf ' enabled: %s\n' "$(yaml_quote "$(env_or_default ENABLE_TELEMETRY "false")")" + printf ' provider: %s\n' "$(yaml_quote "$(env_or_default MONITORING_PROVIDER "otlp")")" + printf ' projectName: %s\n' "$(yaml_quote "$(env_or_default MONITORING_PROJECT_NAME "")")" + printf ' serviceName: %s\n' "$(yaml_quote "$(env_or_default OTEL_SERVICE_NAME "nexent-backend")")" + printf ' otlpEndpoint: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_ENDPOINT "http://nexent-otel-collector:4318")")" + printf ' otlpTracesEndpoint: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_TRACES_ENDPOINT "")")" + printf ' otlpMetricsEndpoint: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_METRICS_ENDPOINT "")")" + printf ' otlpProtocol: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_PROTOCOL "http")")" + printf ' otlpHeaders: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_HEADERS "")")" + printf ' otlpAuthorization: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_AUTHORIZATION "")")" + printf ' otlpApiKey: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_X_API_KEY "")")" + printf ' otlpLangfuseIngestionVersion: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_LANGFUSE_INGESTION_VERSION "")")" + printf ' langsmithApiKey: %s\n' "$(yaml_quote "$(env_or_default LANGSMITH_API_KEY "")")" + printf ' langsmithProject: %s\n' "$(yaml_quote "$(env_or_default LANGSMITH_PROJECT "")")" + printf ' otlpMetricsEnabled: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_METRICS_ENABLED "true")")" + printf ' instrumentRequests: %s\n' "$(yaml_quote "$(env_or_default MONITORING_INSTRUMENT_REQUESTS "false")")" + printf ' fastapiIncludedUrls: %s\n' "$(yaml_quote "$(env_or_default MONITORING_FASTAPI_INCLUDED_URLS "")")" + printf ' fastapiExcludedUrls: %s\n' "$(yaml_quote "$(env_or_default MONITORING_FASTAPI_EXCLUDED_URLS "")")" + printf ' fastapiExcludeSpans: %s\n' "$(yaml_quote "$(env_or_default MONITORING_FASTAPI_EXCLUDE_SPANS "receive,send")")" + printf ' dashboardUrl: %s\n' "$(yaml_quote "$(env_or_default MONITORING_DASHBOARD_URL "")")" + printf ' telemetrySampleRate: %s\n' "$(yaml_quote "$(env_or_default TELEMETRY_SAMPLE_RATE "1.0")")" + printf ' traceContentMode: %s\n' "$(yaml_quote "$(env_or_default MONITORING_TRACE_CONTENT_MODE "full")")" + printf ' traceMaxChars: %s\n' "$(yaml_quote "$(env_or_default MONITORING_TRACE_MAX_CHARS "4000")")" + printf ' traceMaxItems: %s\n' "$(yaml_quote "$(env_or_default MONITORING_TRACE_MAX_ITEMS "20")")" + echo " oauth:" + printf ' githubClientId: %s\n' "$(yaml_quote "$(env_or_default GITHUB_OAUTH_CLIENT_ID "")")" + printf ' githubClientSecret: %s\n' "$(yaml_quote "$(env_or_default GITHUB_OAUTH_CLIENT_SECRET "")")" + printf ' enableWechat: %s\n' "$(yaml_quote "$(env_or_default ENABLE_WECHAT_OAUTH "false")")" + printf ' wechatClientId: %s\n' "$(yaml_quote "$(env_or_default WECHAT_OAUTH_APP_ID "")")" + printf ' wechatClientSecret: %s\n' "$(yaml_quote "$(env_or_default WECHAT_OAUTH_APP_SECRET "")")" + printf ' gdeUrl: %s\n' "$(yaml_quote "$(env_or_default GDE_URL "")")" + printf ' gdeClientId: %s\n' "$(yaml_quote "$(env_or_default GDE_OAUTH_CLIENT_ID "")")" + printf ' gdeClientSecret: %s\n' "$(yaml_quote "$(env_or_default GDE_OAUTH_CLIENT_SECRET "")")" + printf ' sslVerify: %s\n' "$(yaml_quote "$(env_or_default OAUTH_SSL_VERIFY "true")")" + printf ' caBundle: %s\n' "$(yaml_quote "$(env_or_default OAUTH_CA_BUNDLE "")")" + printf ' callbackBaseUrl: %s\n' "$(yaml_quote "$(env_or_default OAUTH_CALLBACK_BASE_URL "http://localhost:30000")")" + echo " cas:" + printf ' enabled: %s\n' "$(yaml_quote "$(env_or_default CAS_ENABLED "false")")" + printf ' serverUrl: %s\n' "$(yaml_quote "$(env_or_default CAS_SERVER_URL "")")" + printf ' validatePath: %s\n' "$(yaml_quote "$(env_or_default CAS_VALIDATE_PATH "/p3/serviceValidate")")" + printf ' callbackBaseUrl: %s\n' "$(yaml_quote "$(env_or_default CAS_CALLBACK_BASE_URL "http://localhost:30000")")" + printf ' loginMode: %s\n' "$(yaml_quote "$(env_or_default CAS_LOGIN_MODE "disabled")")" + printf ' userAttribute: %s\n' "$(yaml_quote "$(env_or_default CAS_USER_ATTRIBUTE "")")" + printf ' emailAttribute: %s\n' "$(yaml_quote "$(env_or_default CAS_EMAIL_ATTRIBUTE "email")")" + printf ' roleAttribute: %s\n' "$(yaml_quote "$(env_or_default CAS_ROLE_ATTRIBUTE "role")")" + printf ' tenantAttribute: %s\n' "$(yaml_quote "$(env_or_default CAS_TENANT_ATTRIBUTE "tenant_id")")" + printf ' roleMapJson: %s\n' "$(yaml_quote "$(env_or_default CAS_ROLE_MAP_JSON "")")" + printf ' sessionMaxAgeSeconds: %s\n' "$(yaml_quote "$(env_or_default CAS_SESSION_MAX_AGE_SECONDS "3600")")" + printf ' localSessionMaxAgeSeconds: %s\n' "$(yaml_quote "$(env_or_default LOCAL_SESSION_MAX_AGE_SECONDS "3600")")" + printf ' renewBeforeSeconds: %s\n' "$(yaml_quote "$(env_or_default CAS_RENEW_BEFORE_SECONDS "300")")" + printf ' renewTimeoutSeconds: %s\n' "$(yaml_quote "$(env_or_default CAS_RENEW_TIMEOUT_SECONDS "10")")" + printf ' syntheticEmailDomain: %s\n' "$(yaml_quote "$(env_or_default CAS_SYNTHETIC_EMAIL_DOMAIN "cas.local")")" + printf ' logoutUrl: %s\n' "$(yaml_quote "$(env_or_default CAS_LOGOUT_URL "")")" + printf ' sslVerify: %s\n' "$(yaml_quote "$(env_or_default CAS_SSL_VERIFY "true")")" + printf ' caBundle: %s\n' "$(yaml_quote "$(env_or_default CAS_CA_BUNDLE "")")" + + } > "$output_file" +} + +# Get APP_VERSION from backend/consts/const.py +get_app_version() { + if declare -F deployment_read_version >/dev/null 2>&1; then + deployment_read_version "" + return 0 + fi + + if [ ! -f "$CONST_FILE" ]; then + echo "" + return + fi + local line + line=$(grep -E 'APP_VERSION' "$CONST_FILE" | tail -n 1 || true) + line="${line##*=}" + line="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + local value + value="$(printf "%s" "$line" | tr -d '"' | tr -d "'")" + echo "$value" +} + +# Persist deployment options to file +persist_deploy_options() { + { + echo "APP_VERSION=\"${APP_VERSION}\"" + echo "IS_MAINLAND=\"${IS_MAINLAND_SAVED}\"" + echo "DEPLOYMENT_VERSION=\"${VERSION_CHOICE_SAVED}\"" + } > "$DEPLOY_OPTIONS_FILE" +} + +# Load deployment options from file if exists +load_deploy_options() { + if [ -f "$DEPLOY_OPTIONS_FILE" ]; then + source "$DEPLOY_OPTIONS_FILE" + fi +} + +# Choose image environment (mainland China or general) +choose_image_env() { + echo "==========================================" + echo " Image Source Selection" + echo "==========================================" + + if [ -n "$IS_MAINLAND" ]; then + is_mainland="$IS_MAINLAND" + echo "Using is_mainland from argument: $is_mainland" + else + load_deploy_options + if [ -n "$IS_MAINLAND" ]; then + is_mainland="$IS_MAINLAND" + echo "Using saved is_mainland: $is_mainland" + else + read -p "Is your server network located in mainland China? [Y/N] (default N): " is_mainland + fi + fi + + is_mainland=$(sanitize_input "$is_mainland") + if [[ "$is_mainland" =~ ^[Yy]$ ]]; then + IS_MAINLAND_SAVED="Y" + echo "Detected mainland China network, using image-source.mainland.env for image sources." + source "$DEPLOY_ROOT/env/image-source.mainland.env" + else + IS_MAINLAND_SAVED="N" + echo "Using general image sources from image-source.general.env." + source "$DEPLOY_ROOT/env/image-source.general.env" + fi + + echo "" + echo "--------------------------------" + echo "" +} + +# Render image tags into generated Helm values based on loaded environment variables +update_values_yaml() { + echo "==========================================" + echo " Rendering generated image values" + echo "==========================================" + + # Get APP_VERSION if not already set + if [ -z "$APP_VERSION" ]; then + APP_VERSION=$(get_app_version) + fi + + if [ -z "$APP_VERSION" ]; then + echo "Failed to determine APP_VERSION from const.py, using 'latest'" + APP_VERSION="latest" + fi + echo "Using APP_VERSION: $APP_VERSION" + echo "" + + deployment_apply_image_source + deployment_render_helm_values "$GENERATED_VALUES" + render_k8s_runtime_config_values "$GENERATED_RUNTIME_VALUES" + render_persistence_values "$GENERATED_PERSISTENCE_VALUES" + echo "Generated Helm values: $GENERATED_VALUES" + echo "Generated Helm runtime values: $GENERATED_RUNTIME_VALUES" + echo "Generated Helm persistence values: $GENERATED_PERSISTENCE_VALUES" + echo "" + echo "--------------------------------" + echo "" +} + +ensure_namespace() { + if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + echo "Namespace '$NAMESPACE' already exists." + else + echo "Creating namespace '$NAMESPACE'..." + kubectl create namespace "$NAMESPACE" + fi +} + +helm_upgrade_release() { + helm upgrade --install nexent "$CHART_DIR" \ + --namespace "$NAMESPACE" \ + -f "$GENERATED_VALUES" \ + -f "$GENERATED_RUNTIME_VALUES" \ + -f "$GENERATED_PERSISTENCE_VALUES" \ + -f "$GENERATED_SECRETS_VALUES" \ + --set nexent-openssh.enabled="$ENABLE_OPENSSH" \ + --set nexent-common.secrets.ssh.username="$SSH_USERNAME" \ + --set nexent-common.secrets.ssh.password="$SSH_PASSWORD" +} + +wait_for_deployment_ready() { + local deployment="$1" + kubectl rollout status "deployment/${deployment}" -n "$NAMESPACE" --timeout="${K8S_WAIT_TIMEOUT_SECONDS}s" +} + +recreate_legacy_nexent_secret_for_helm_management() { + local managers + if ! kubectl get secret nexent-secrets -n "$NAMESPACE" >/dev/null 2>&1; then + return 0 + fi + + managers=$(kubectl get secret nexent-secrets -n "$NAMESPACE" -o jsonpath='{range .metadata.managedFields[*]}{.manager}{"\n"}{end}' 2>/dev/null || true) + if printf '%s\n' "$managers" | grep -qx 'kubectl-patch'; then + echo "Recreating legacy nexent-secrets so Helm owns all Secret fields..." + kubectl delete secret nexent-secrets -n "$NAMESPACE" + fi +} + +# Select deployment version (speed or full) +select_deployment_version() { + echo "==========================================" + echo " Deployment Version Selection" + echo "==========================================" + echo "Please select deployment version:" + echo " 1) Speed version - Lightweight deployment with essential features (no Supabase)" + echo " 2) Full version - Full-featured deployment with all capabilities (includes Supabase)" + + if [ -n "$DEPLOYMENT_VERSION" ]; then + version_choice="$DEPLOYMENT_VERSION" + echo "Using deployment-version from argument: $version_choice" + else + load_deploy_options + if [ -n "$DEPLOYMENT_VERSION" ]; then + version_choice="$DEPLOYMENT_VERSION" + echo "Using saved deployment-version: $version_choice" + else + read -p "Enter your choice [1/2] (default: 1): " version_choice + fi + fi + + version_choice=$(sanitize_input "$version_choice") + VERSION_CHOICE_SAVED="${version_choice}" + + case $version_choice in + 2|"full") + export DEPLOYMENT_VERSION="full" + echo "Selected complete version" + ;; + 1|"speed"|*) + export DEPLOYMENT_VERSION="speed" + echo "Selected speed version" + ;; + esac + + # Legacy helper retained for compatibility; generated values carry the effective version. + + echo "" + echo "--------------------------------" + echo "" +} + +# Generate JWT token for Supabase +generate_jwt() { + local role=$1 + local secret=$JWT_SECRET + local now=$(date +%s) + local exp=$((now + 157680000)) + + local header='{"alg":"HS256","typ":"JWT"}' + local header_base64=$(echo -n "$header" | base64 | tr -d '\n=' | tr '/+' '_-') + + local payload="{\"role\":\"$role\",\"iss\":\"supabase\",\"iat\":$now,\"exp\":$exp}" + local payload_base64=$(echo -n "$payload" | base64 | tr -d '\n=' | tr '/+' '_-') + + local signature=$(echo -n "$header_base64.$payload_base64" | openssl dgst -sha256 -hmac "$secret" -binary | base64 | tr -d '\n=' | tr '/+' '_-') + + echo "$header_base64.$payload_base64.$signature" +} + +decode_base64() { + if base64 --help 2>&1 | grep -q -- '--decode'; then + base64 --decode + else + base64 -D + fi +} + +get_existing_secret_value() { + local key="$1" + local encoded_value + encoded_value=$(kubectl get secret nexent-secrets -n "$NAMESPACE" -o jsonpath="{.data.${key}}" 2>/dev/null || true) + if [ -z "$encoded_value" ]; then + return 1 + fi + + printf '%s' "$encoded_value" | decode_base64 +} + +load_existing_supabase_secrets() { + local existing_jwt_secret + local existing_secret_key_base + local existing_vault_enc_key + local existing_anon_key + local existing_service_role_key + + existing_jwt_secret="$(get_existing_secret_value "JWT_SECRET")" || return 1 + existing_secret_key_base="$(get_existing_secret_value "SECRET_KEY_BASE")" || return 1 + existing_vault_enc_key="$(get_existing_secret_value "VAULT_ENC_KEY")" || return 1 + existing_anon_key="$(get_existing_secret_value "SUPABASE_KEY")" || return 1 + existing_service_role_key="$(get_existing_secret_value "SERVICE_ROLE_KEY")" || return 1 + + JWT_SECRET="$existing_jwt_secret" + SECRET_KEY_BASE="$existing_secret_key_base" + VAULT_ENC_KEY="$existing_vault_enc_key" + SUPABASE_ANON_KEY="$existing_anon_key" + SUPABASE_SERVICE_ROLE_KEY="$existing_service_role_key" + return 0 +} + +load_existing_minio_secrets() { + local existing_access_key + local existing_secret_key + + existing_access_key="$(get_existing_secret_value "MINIO_ACCESS_KEY")" || return 1 + existing_secret_key="$(get_existing_secret_value "MINIO_SECRET_KEY")" || return 1 + + if [ -z "$existing_access_key" ] || [ -z "$existing_secret_key" ]; then + return 1 + fi + + MINIO_ACCESS_KEY="$existing_access_key" + MINIO_SECRET_KEY="$existing_secret_key" + return 0 +} + +load_existing_elasticsearch_api_key() { + local existing_api_key + existing_api_key="$(get_existing_secret_value "ELASTICSEARCH_API_KEY")" || return 1 + [ -n "$existing_api_key" ] || return 1 + ELASTICSEARCH_API_KEY="$existing_api_key" + return 0 +} + +# Generate Supabase secrets (only for full version) +generate_supabase_secrets() { + if [ "$DEPLOYMENT_VERSION" != "full" ]; then + echo "Skipping Supabase secrets generation (deployment version is speed)" + return 0 + fi + + echo "==========================================" + echo " Supabase Secrets Generation" + echo "==========================================" + + if [ -n "${JWT_SECRET:-}" ] && [ -n "${SECRET_KEY_BASE:-}" ] && [ -n "${VAULT_ENC_KEY:-}" ] && [ -n "${SUPABASE_KEY:-}" ] && [ -n "${SERVICE_ROLE_KEY:-}" ]; then + SUPABASE_ANON_KEY="$SUPABASE_KEY" + SUPABASE_SERVICE_ROLE_KEY="$SERVICE_ROLE_KEY" + echo "Using Supabase secrets from root .env." + echo "" + echo "--------------------------------" + echo "" + return 0 + fi + + if load_existing_supabase_secrets; then + echo "Reusing existing Supabase secrets from Kubernetes secret." + echo "" + echo "--------------------------------" + echo "" + return 0 + fi + + # Generate fresh keys for security + JWT_SECRET=$(openssl rand -base64 32 | tr -d '[:space:]') + SECRET_KEY_BASE=$(openssl rand -base64 64 | tr -d '[:space:]') + VAULT_ENC_KEY=$(openssl rand -base64 32 | tr -d '[:space:]') + + # Generate JWT-dependent keys + local anon_key=$(generate_jwt "anon") + local service_role_key=$(generate_jwt "service_role") + + SUPABASE_ANON_KEY="$anon_key" + SUPABASE_SERVICE_ROLE_KEY="$service_role_key" + echo "Supabase secrets generated for generated Helm values" + echo "" + echo "--------------------------------" + echo "" +} + +# Pull MCP Docker image to local host (best-effort) +pull_mcp_image() { + echo "==========================================" + echo " MCP Image Pull" + echo "==========================================" + + # Use image from environment, fallback to default image + local image="${NEXENT_MCP_DOCKER_IMAGE:-nexent/nexent-mcp}" + local image_tail="${image##*/}" + local mcp_image_name="$image" + if [[ "$image_tail" != *:* ]]; then + mcp_image_name="${image}:${APP_VERSION:-latest}" + fi + echo "Checking MCP image: ${mcp_image_name}" + + if ! command -v docker >/dev/null 2>&1; then + echo "Warning: Docker is not installed or not in PATH, skipping MCP image pull." + echo "" + echo "--------------------------------" + echo "" + return 0 + fi + + # Pull image only when not present locally + if docker image inspect "${mcp_image_name}" >/dev/null 2>&1; then + echo "MCP image already exists locally, skipping pull." + elif [ "$DEPLOYMENT_IMAGE_SOURCE" = "local-latest" ]; then + echo "Warning: MCP local image not found: ${mcp_image_name}" + echo "Build or load it locally before using --image-source local-latest." + else + echo "MCP image not found locally, pulling..." + if docker pull "${mcp_image_name}"; then + echo "MCP image pulled successfully." + else + echo "Warning: Failed to pull MCP image, but deployment will continue." + echo "You can pull it manually later: docker pull ${mcp_image_name}" + fi + fi + + echo "" + echo "--------------------------------" + echo "" +} + +render_runtime_secret_values() { + local gotrue_db_url + local runtime_config_hash + local backend_checksum + local minio_checksum + local supabase_checksum + local web_checksum + local ssh_checksum + local sql_checksum + + gotrue_db_url="$(env_or_default GOTRUE_DB_DATABASE_URL "postgres://supabase_auth_admin:$(env_or_default SUPABASE_POSTGRES_PASSWORD "Huawei123")@$(env_or_default SUPABASE_POSTGRES_HOST "nexent-supabase-db"):$(env_or_default SUPABASE_POSTGRES_PORT "5436")/$(env_or_default SUPABASE_POSTGRES_DB "supabase")?search_path=auth&sslmode=disable")" + runtime_config_hash="$(deployment_sha256_file "$GENERATED_RUNTIME_VALUES")" + sql_checksum="$(sql_files_checksum)" + backend_checksum="$(deployment_sha256_string "runtime=${runtime_config_hash}|sql=${sql_checksum}|elastic=$(env_or_default ELASTICSEARCH_API_KEY "")|postgres=$(env_or_default NEXENT_POSTGRES_PASSWORD "nexent@4321")|minio=${MINIO_ACCESS_KEY}:${MINIO_SECRET_KEY}")" + minio_checksum="$(deployment_sha256_string "root=$(env_or_default MINIO_ROOT_USER "nexent"):$(env_or_default MINIO_ROOT_PASSWORD "nexent@4321")|client=${MINIO_ACCESS_KEY}:${MINIO_SECRET_KEY}")" + supabase_checksum="$(deployment_sha256_string "jwt=${JWT_SECRET:-}|base=${SECRET_KEY_BASE:-}|vault=${VAULT_ENC_KEY:-}|anon=${SUPABASE_ANON_KEY:-}|service=${SUPABASE_SERVICE_ROLE_KEY:-}|pg=$(env_or_default SUPABASE_POSTGRES_PASSWORD "Huawei123")|db=${gotrue_db_url}")" + web_checksum="$(deployment_sha256_string "market=$(env_or_default MARKET_BACKEND "http://60.204.251.153:8010")|model=$(env_or_default MODEL_ENGINE_ENABLED "false")")" + ssh_checksum="$(deployment_sha256_string "ssh=$(env_or_default SSH_USERNAME "nexent"):$(env_or_default SSH_PASSWORD "nexent@2025")")" + + { + echo "global:" + echo " rolloutChecksums:" + printf ' backend: %s\n' "$(yaml_quote "$backend_checksum")" + printf ' minio: %s\n' "$(yaml_quote "$minio_checksum")" + printf ' supabase: %s\n' "$(yaml_quote "$supabase_checksum")" + printf ' web: %s\n' "$(yaml_quote "$web_checksum")" + printf ' ssh: %s\n' "$(yaml_quote "$ssh_checksum")" + printf ' sql: %s\n' "$(yaml_quote "$sql_checksum")" + echo "nexent-common:" + echo " secrets:" + printf ' elasticPassword: %s\n' "$(yaml_quote "$(env_or_default ELASTIC_PASSWORD "nexent@2025")")" + printf ' elasticsearchApiKey: %s\n' "$(yaml_quote "$(env_or_default ELASTICSEARCH_API_KEY "")")" + printf ' postgresPassword: %s\n' "$(yaml_quote "$(env_or_default NEXENT_POSTGRES_PASSWORD "nexent@4321")")" + echo " minio:" + printf ' rootUser: %s\n' "$(yaml_quote "$(env_or_default MINIO_ROOT_USER "nexent")")" + printf ' rootPassword: %s\n' "$(yaml_quote "$(env_or_default MINIO_ROOT_PASSWORD "nexent@4321")")" + printf ' accessKey: %s\n' "$(yaml_quote "$MINIO_ACCESS_KEY")" + printf ' secretKey: %s\n' "$(yaml_quote "$MINIO_SECRET_KEY")" + echo " ssh:" + printf ' username: %s\n' "$(yaml_quote "$(env_or_default SSH_USERNAME "nexent")")" + printf ' password: %s\n' "$(yaml_quote "$(env_or_default SSH_PASSWORD "nexent@2025")")" + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "supabase"; then + echo " supabase:" + printf ' jwtSecret: %s\n' "$(yaml_quote "$JWT_SECRET")" + printf ' secretKeyBase: %s\n' "$(yaml_quote "$SECRET_KEY_BASE")" + printf ' vaultEncKey: %s\n' "$(yaml_quote "$VAULT_ENC_KEY")" + printf ' anonKey: %s\n' "$(yaml_quote "$SUPABASE_ANON_KEY")" + printf ' serviceRoleKey: %s\n' "$(yaml_quote "$SUPABASE_SERVICE_ROLE_KEY")" + printf ' postgresPassword: %s\n' "$(yaml_quote "$(env_or_default SUPABASE_POSTGRES_PASSWORD "Huawei123")")" + printf ' gotrueDbUrl: %s\n' "$(yaml_quote "$gotrue_db_url")" + fi + } > "$GENERATED_SECRETS_VALUES" +} + +apply() { + echo "Deploying Nexent using Helm..." + + # Step 1: Select deployment components, port policy and image source. + apply_deployment_common_config + deployment_persist_local_config + + # Step 2: Render generated values with image tags from selected environment + update_values_yaml + + # Step 3: Generate MinIO Access Key and Secret Key + echo "==========================================" + echo " MinIO Access Key/Secret Key Setup" + echo "==========================================" + if [ -n "${MINIO_ACCESS_KEY:-}" ] && [ -n "${MINIO_SECRET_KEY:-}" ]; then + echo "Using MinIO credentials from root .env." + echo "Access Key: $MINIO_ACCESS_KEY" + elif load_existing_minio_secrets; then + echo "Reusing existing MinIO credentials from Kubernetes secret." + echo "Access Key: $MINIO_ACCESS_KEY" + elif grep -q "minio:" "$COMMON_VALUES" && grep -q "accessKey:" "$COMMON_VALUES"; then + MINIO_ACCESS_KEY=$(grep "accessKey:" "$COMMON_VALUES" | head -1 | sed 's/.*accessKey: *//' | tr -d '"' | tr -d "'" | xargs) + MINIO_SECRET_KEY=$(grep "secretKey:" "$COMMON_VALUES" | head -1 | sed 's/.*secretKey: *//' | tr -d '"' | tr -d "'" | xargs) + fi + + if [ -z "$MINIO_ACCESS_KEY" ] || [ "$MINIO_ACCESS_KEY" = "" ]; then + echo "Generating new MinIO Access Key and Secret Key..." + MINIO_ACCESS_KEY="nexent-$(head -c 8 /dev/urandom | base64 | tr -dc 'a-z0-9' | head -c 12)" + MINIO_SECRET_KEY=$(head -c 32 /dev/urandom | base64 | tr -dc 'A-Za-z0-9' | head -c 24) + + echo "MinIO credentials generated for generated Helm values" + echo "Access Key: $MINIO_ACCESS_KEY" + echo "Secret Key: $MINIO_SECRET_KEY (saved in generated Helm values)" + else + echo "MinIO credentials already exist in chart defaults" + echo "Access Key: $MINIO_ACCESS_KEY" + fi + echo "" + + # Step 4: Generate Supabase secrets (only for full version) + generate_supabase_secrets + + if [ "${DEPLOYMENT_REFRESH_ES_KEY:-false}" != "true" ] && [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" != "true" ]; then + if [ -n "${ELASTICSEARCH_API_KEY:-}" ]; then + echo "Using ELASTICSEARCH_API_KEY from root .env." + elif load_existing_elasticsearch_api_key; then + echo "Reusing existing ELASTICSEARCH_API_KEY from Kubernetes secret." + fi + fi + + render_runtime_secret_values + + # Step 5: Configure Terminal tool (OpenSSH) only when selected. + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "terminal"; then + ENABLE_OPENSSH="true" + echo "Terminal tool will be enabled." + + # Ask for SSH credentials + echo "" + echo "SSH credentials configuration:" + read -p "SSH Username (default: nexent): " ssh_username + SSH_USERNAME="${ssh_username:-nexent}" + read -s -p "SSH Password (default: nexent@2025): " ssh_password + echo "" + SSH_PASSWORD="${ssh_password:-nexent@2025}" + else + ENABLE_OPENSSH="false" + echo "Terminal tool disabled." + fi + echo "" + + # Step 6: Clean up stale PVs + echo "Checking for stale PersistentVolumes..." + for pv in nexent-workspace-pv nexent-skills-pv nexent-elasticsearch-pv nexent-postgresql-pv nexent-redis-pv nexent-minio-pv; do + pv_status=$(kubectl get pv $pv -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound") + if [ "$pv_status" = "Released" ]; then + echo " Cleaning up stale PV: $pv" + kubectl delete pv $pv --ignore-not-found=true || true + fi + done + + # Clean up supabase PV if exists + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "supabase"; then + for pv in nexent-supabase-db-pv; do + pv_status=$(kubectl get pv $pv -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound") + if [ "$pv_status" = "Released" ]; then + echo " Cleaning up stale PV: $pv" + kubectl delete pv $pv --ignore-not-found=true || true + fi + done + fi + + # Step 7: Deploy using Helm + ensure_namespace + recreate_legacy_nexent_secret_for_helm_management + echo "Deploying Helm chart..." + helm_upgrade_release + + # Step 9: Wait for Elasticsearch to be ready and initialize API key + echo "" + echo "==========================================" + echo " Elasticsearch Initialization" + echo "==========================================" + local deploy_success=true + + echo "Waiting for Elasticsearch deployment to be ready..." + sleep 5 + if wait_for_deployment_ready "nexent-elasticsearch"; then + echo "Elasticsearch deployment is ready." + + # Initialize Elasticsearch API key only when it is missing, invalid, or explicitly refreshed. + INIT_ES_SCRIPT="$SCRIPT_DIR/init-elasticsearch.sh" + if [ -f "$INIT_ES_SCRIPT" ]; then + echo "Running Elasticsearch initialization script..." + local es_key_before + local es_key_after + local es_key_output_file + es_key_before="$(get_existing_secret_value "ELASTICSEARCH_API_KEY" || true)" + es_key_output_file="$(mktemp "${TMPDIR:-/tmp}/nexent-es-key.XXXXXX")" + if ROOT_ENV_FILE="$ROOT_ENV_FILE" ELASTICSEARCH_API_KEY_OUTPUT_FILE="$es_key_output_file" DEPLOYMENT_REFRESH_ES_KEY="${DEPLOYMENT_REFRESH_ES_KEY:-false}" DEPLOYMENT_ROTATE_SECRETS="${DEPLOYMENT_ROTATE_SECRETS:-false}" bash "$INIT_ES_SCRIPT"; then + if [ -s "$es_key_output_file" ]; then + es_key_after="$(cat "$es_key_output_file")" + else + es_key_after="$es_key_before" + fi + rm -f "$es_key_output_file" + echo "Elasticsearch API key initialized successfully." + + if [ "$es_key_before" != "$es_key_after" ]; then + echo "" + echo "ELASTICSEARCH_API_KEY updated; refreshing Helm values and rolling affected backend services..." + ELASTICSEARCH_API_KEY="$es_key_after" + render_runtime_secret_values + helm_upgrade_release + + local backend_services="config runtime mcp northbound" + deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "data-process" && backend_services="$backend_services data-process" + + echo "" + echo "Waiting for backend services to be ready..." + sleep 5 + for svc in $backend_services; do + echo " Waiting for nexent-$svc..." + if wait_for_deployment_ready "nexent-$svc"; then + echo " nexent-$svc is ready." + else + echo " Error: nexent-$svc did not become ready within ${K8S_WAIT_TIMEOUT_SECONDS}s." + deploy_success=false + fi + done + else + echo "ELASTICSEARCH_API_KEY unchanged; backend rollout is not needed." + fi + else + rm -f "$es_key_output_file" + echo "Error: Elasticsearch initialization script failed." + deploy_success=false + fi + else + echo "Error: init-elasticsearch.sh not found at $INIT_ES_SCRIPT" + deploy_success=false + fi + else + echo "Error: nexent-elasticsearch did not become ready within ${K8S_WAIT_TIMEOUT_SECONDS}s." + deploy_success=false + fi + + if [ "$deploy_success" = false ]; then + echo "" + echo "==========================================" + echo " Deployment Failed!" + echo "==========================================" + exit 1 + fi + + # Step 10: Create super admin user (only for full deployment) + CREATE_SUADMIN_SCRIPT="$SCRIPT_DIR/create-suadmin.sh" + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "supabase"; then + if [ -f "$CREATE_SUADMIN_SCRIPT" ]; then + echo "" + echo "==========================================" + echo " Super Admin User Creation" + echo "==========================================" + if bash "$CREATE_SUADMIN_SCRIPT"; then + echo "Super admin user creation completed." + else + echo "Warning: Super admin user creation failed, but continuing deployment." + fi + else + echo "Warning: create-suadmin.sh not found at $CREATE_SUADMIN_SCRIPT" + fi + fi + + # Save deployment options for future use + persist_deploy_options + deployment_persist_local_config + + # Step 11: Pull MCP image after persisting deployment options + pull_mcp_image + + echo "Deployment completed successfully!" + echo "Access the application at: http://localhost:30000" + if [ "$ENABLE_OPENSSH" = "true" ]; then + echo "SSH Terminal at: localhost:30022" + fi +} + +print_usage() { + echo "Usage: $0 [apply] [options]" + echo "" + echo "Deploy Nexent K8s resources using Helm." + echo "" + echo "Options:" + echo " --components LIST Components to deploy" + echo " --port-policy POLICY development or production" + echo " --image-source SOURCE general, mainland, or local-latest" + echo " --is-mainland Y|N Legacy alias for image source mainland/general" + echo " --version VERSION Specify app version (auto-detected from const.py if not set)" + echo " --deployment-version VER Legacy deployment version: speed or full" + echo " --persistence-mode MODE local, dynamic, or existing" + echo " --storage-class NAME StorageClass for PV/PVC binding (aliases: --storageclass, --storage-class-name, --sc)" + echo " --local-path PATH Base path for local PVs" + echo " --local-node-name NAME Deprecated; local mode uses hostPath and does not require nodeAffinity" + echo " --existing-claim-prefix P Existing PVC prefix, rendered as P-" + echo " --wait-timeout SECONDS Kubernetes deployment wait timeout (default: 600)" + echo " --rotate-secrets Force rotation of deployment secrets" + echo " --refresh-es-key Force recreation of ELASTICSEARCH_API_KEY" + echo " --help, -h Show this help message" + echo "" + echo "Uninstall: bash uninstall.sh" +} + +case "$COMMAND" in +help) + print_usage + ;; +apply) + apply + ;; +esac diff --git a/k8s/helm/nexent/Chart.yaml b/deploy/k8s/helm/nexent/Chart.yaml similarity index 100% rename from k8s/helm/nexent/Chart.yaml rename to deploy/k8s/helm/nexent/Chart.yaml diff --git a/k8s/helm/nexent/README.md b/deploy/k8s/helm/nexent/README.md similarity index 65% rename from k8s/helm/nexent/README.md rename to deploy/k8s/helm/nexent/README.md index 1e74bae41..8845146f3 100644 --- a/k8s/helm/nexent/README.md +++ b/deploy/k8s/helm/nexent/README.md @@ -10,63 +10,69 @@ This directory contains a Helm chart for deploying Nexent on Kubernetes. ## Quick Start -Navigate to the `k8s/helm` directory and run the deployment script: +From the repository root, run the root deployment entrypoint: ```bash -cd k8s/helm -./deploy.sh +bash deploy.sh k8s ``` ## Commands | Command | Description | |---------|-------------| -| `./deploy.sh` | Deploy all K8s resources | -| `./uninstall.sh` | Uninstall the Helm release; prompts before deleting namespace or local data | -| `./uninstall.sh clean` | Clean Helm state only (fixes stuck releases) | -| `./uninstall.sh delete` | Uninstall the Helm release and delete the namespace | -| `./uninstall.sh delete-all` | Uninstall the Helm release, delete the namespace, and delete local hostPath data | +| `bash deploy.sh k8s` | Deploy all K8s resources from the repository root | +| `bash uninstall.sh k8s` | Uninstall the Helm release from the repository root; prompts before deleting namespace or local data | +| `bash uninstall.sh k8s clean` | Clean Helm state only (fixes stuck releases) | +| `bash uninstall.sh k8s delete` | Uninstall the Helm release and delete the namespace | +| `bash uninstall.sh k8s delete-all` | Uninstall the Helm release, delete the namespace, and delete local PV data | ### Usage Examples ```bash # Interactive deployment (will prompt for all options) -./deploy.sh +bash deploy.sh k8s # Non-interactive deployment with the default component set -./deploy.sh --components infrastructure,application --port-policy development --image-source general +bash deploy.sh k8s --components infrastructure,application,data-process,supabase --port-policy development --image-source general -# Enable Supabase, data processing, and terminal -./deploy.sh --components infrastructure,application,supabase,data-process,terminal +# Add terminal to the default component set +bash deploy.sh k8s --components infrastructure,application,data-process,supabase,terminal # Use mainland China image sources -./deploy.sh --image-source mainland +bash deploy.sh k8s --image-source mainland # Use local latest Nexent images -./deploy.sh --image-source local-latest +bash deploy.sh k8s --image-source local-latest + +# Use a specific StorageClass with the short alias +bash deploy.sh k8s --sc fast-storage # Clean helm state (fixes stuck releases) -./uninstall.sh clean +bash uninstall.sh k8s clean # Uninstall but preserve data -./uninstall.sh +bash uninstall.sh k8s -# Uninstall and keep local hostPath data without prompting -./uninstall.sh --keep-local-data --keep-namespace +# Uninstall and keep local PV data without prompting +bash uninstall.sh k8s --keep-local-data --keep-namespace # Delete namespace after uninstall -./uninstall.sh --delete-namespace true +bash uninstall.sh k8s --delete-namespace true -# Delete local hostPath data after uninstall -./uninstall.sh --delete-local-data true +# Delete local PV data after uninstall +bash uninstall.sh k8s --delete-local-data true -# Complete uninstall including namespace and local hostPath data -./uninstall.sh delete-all +# Complete uninstall including namespace and local PV data +bash uninstall.sh k8s delete-all -# Complete uninstall but preserve local hostPath data -./uninstall.sh delete-all --keep-local-data +# Complete uninstall but preserve local PV data +bash uninstall.sh k8s delete-all --keep-local-data ``` +K8s deployments read runtime configuration from the project root `.env`, the same file used by Docker. Existing `.env` is kept as-is. If it is missing, the deploy script first reuses an existing legacy `docker/.env`, then falls back to `.env.example` or `docker/.env.example`. Do not edit generated Helm values by hand; they are recreated from `.env` and deployment options. + +When `--persistence-mode local` is used, Nexent renders static PVs with `hostPath` and `DirectoryOrCreate`; node affinity is not required. Shared workspace data uses `/var/lib/nexent`, shared skills use `/var/lib/nexent-data/skills`, and service data uses `/var/lib/nexent-data/nexent-*` by default. + ## Deploy Options | Option | Description | Values | @@ -82,6 +88,11 @@ cd k8s/helm | `--is-mainland` | Legacy network location option | `Y` maps to `--image-source mainland`; `N` maps to `general` | | `--version` | Application version | Version tag (auto-detected from `backend/consts/const.py` if not set) | | `--deployment-version` | Legacy deployment version | `speed` maps to `infrastructure,application`; `full` adds `supabase` | +| `--persistence-mode` | Persistent volume mode | `local`, `dynamic`, or `existing`; default `local` | +| `--storage-class` | StorageClass for PV/PVC binding | StorageClass name; aliases `--storageclass`, `--storage-class-name`, `--sc` | +| `--local-path` | Base host path for local PVs except workspace | Path; default `/var/lib/nexent-data` | +| `--local-node-name` | Deprecated compatibility option | Ignored; local mode uses hostPath and does not require nodeAffinity | +| `--existing-claim-prefix` | Prefix for existing PVC names | Renders as `-` | ## Uninstall Options @@ -91,7 +102,7 @@ cd k8s/helm | `--delete-volumes` | Alias for `--delete-data` | `true` or `false` | | `--remove-volumes` | Alias for `--delete-data true` | Flag | | `--keep-volumes` | Alias for `--delete-data false` | Flag | -| `--delete-local-data` | Delete local hostPath data under `/var/lib/nexent-data` after Helm uninstall | `true` or `false` | +| `--delete-local-data` | Delete local PV data under `/var/lib/nexent` and `/var/lib/nexent-data` after Helm uninstall | `true` or `false` | | `--remove-local-data` | Alias for `--delete-local-data true` | Flag | | `--keep-local-data` | Alias for `--delete-local-data false` | Flag | | `--delete-namespace` | Delete the Kubernetes namespace after Helm uninstall | `true` or `false` | @@ -100,9 +111,38 @@ cd k8s/helm | `--namespace` | Kubernetes namespace | Namespace name; default `nexent` | | `--release` | Helm release name | Release name; default `nexent` | +## Offline Image Package + +Use the repository-level offline package builder when the target Kubernetes environment cannot pull images directly: + +```bash +bash deploy/offline/build_offline_package.sh \ + --target k8s \ + --version v2.2.1 \ + --platform amd64 \ + --components infrastructure,application,data-process,supabase \ + --image-source general \ + --compress true \ + --output-dir offline-package/k8s +``` + +Package contents include `images/*.tar`, `load-images.sh`, root `deploy.sh` and `uninstall.sh`, the filtered `deploy/` bundle for the selected target, `deploy/sql`, `manifest.yaml`, and `checksums.txt`. Local `.env`, `.env.generated`, and `deploy.options` are intentionally excluded. With `--compress true`, a `nexent-offline---.zip` archive is created next to the output directory. + +On a target host with access to the cluster, load images before deployment: + +```bash +cd offline-package/k8s +bash deploy.sh --load-images k8s \ + --version v2.2.1 \ + --components infrastructure,application,data-process,supabase \ + --image-source general +``` + +For multi-node clusters, run `load-images.sh` on every node that may schedule Nexent Pods, or push the loaded images to an internal registry and deploy with matching image references. + ## Deployment Components -The deployment script uses Bash TUI menus when running interactively. It first shows a component multi-select menu, then single-select menus for port policy and image source. Use `b`/Backspace to return to the previous TUI step and `q` to quit. `infrastructure` is required and is added automatically if omitted; `application` is selected by default but can be disabled. +The deployment script uses Bash TUI menus when running interactively. It first shows a component multi-select menu, then single-select menus for port policy and image source. Use `b`/Backspace to return to the previous TUI step and `q` to quit. `infrastructure` is required and is added automatically if omitted; `application`, `data-process`, and `supabase` are selected by default and can be disabled for smaller deployments. | Component | Services | |-----------|----------| @@ -113,7 +153,7 @@ The deployment script uses Bash TUI menus when running interactively. It first s | `terminal` | OpenSSH terminal tool | | `monitoring` | Optional monitoring chart; selecting it prompts for provider unless `--monitoring-provider` is passed | -`application` does not include `data-process`. User and tenant features are enabled by selecting `supabase`; there is no separate user/tenant switch. +`application` does not include `data-process`; it is a separate component even though it is selected by default. User and tenant features are enabled by selecting `supabase`; there is no separate user/tenant switch. ## Port Policy @@ -147,7 +187,7 @@ Image source is independent from components and ports: - `mainland`: uses mainland China registry mirror images and `--version`. - `local-latest`: uses local `latest` Nexent images and sets local-friendly pull policy. -After successful deployment, non-sensitive deployment choices are saved to `k8s/helm/deploy.options`. The next interactive run can reuse that config or reconfigure from scratch. Generated Helm values are runtime files and are ignored by git. +After successful deployment, non-sensitive deployment choices are saved to `deploy/k8s/deploy.options`. The next interactive run can reuse that config or reconfigure from scratch. Generated Helm values are runtime files and are ignored by git. ## Accessing the Application @@ -166,10 +206,12 @@ After successful deployment: ### Preserved Data -By default, `./uninstall.sh` removes the Helm release and preserves local hostPath data. It prompts before deleting the namespace or hostPath contents. In non-interactive environments, both are preserved unless explicitly requested. +By default, `bash uninstall.sh k8s` removes the Helm release and preserves local PV data. It prompts before deleting the namespace or local PV contents. In non-interactive environments, both are preserved unless explicitly requested. -The following local hostPath-backed PersistentVolumes can preserve data: +The following local PersistentVolumes can preserve data: +- `nexent-workspace-pv` - Shared user workspace mounted at `/mnt/nexent` +- `nexent-skills-pv` - Shared skills data mounted at `/mnt/nexent-data/skills` - `nexent-elasticsearch-pv` - Search index data - `nexent-postgresql-pv` - Relational database data - `nexent-redis-pv` - Cache data @@ -179,7 +221,7 @@ The following local hostPath-backed PersistentVolumes can preserve data: ### Deleted Data -Use `--delete-local-data true` or `--remove-local-data` to delete known Nexent hostPath data under `/var/lib/nexent-data/nexent-*`. `delete-all` deletes the namespace and local hostPath data by default; add `--keep-local-data` to preserve local volume contents. +Use `--delete-local-data true` or `--remove-local-data` to delete known Nexent local PV data under `/var/lib/nexent`, `/var/lib/nexent-data/skills`, and `/var/lib/nexent-data/nexent-*`. `delete-all` deletes the namespace and local PV data by default; add `--keep-local-data` to preserve local volume contents. ## Services @@ -286,7 +328,11 @@ helm upgrade --install nexent nexent \ | Parameter | Description | Default | |-----------|-------------|---------| | `global.namespace` | Kubernetes namespace | `nexent` | -| `global.dataDir` | Host path for persistent data | `/data/nexent` | +| `global.dataDir` | Host path for persistent data | `/var/lib/nexent-data` | +| `global.sharedStorage.workspace.size` | Shared `/mnt/nexent` PVC size | `10Gi` | +| `global.sharedStorage.workspace.localPath` | Host path for shared workspace data | `/var/lib/nexent` | +| `global.sharedStorage.skills.size` | Shared `/mnt/nexent-data/skills` PVC size | `5Gi` | +| `global.sharedStorage.skills.localPath` | Host path for shared skills data | `/var/lib/nexent-data/skills` | | `deploymentVersion` | Deployment version | `speed` | #### Images @@ -330,8 +376,8 @@ helm upgrade --install nexent nexent \ If you see "Release does not exist" errors: ```bash -./uninstall.sh clean -./deploy.sh +bash uninstall.sh k8s clean +bash deploy.sh k8s ``` ### Pods Not Starting @@ -355,8 +401,7 @@ kubectl logs -n nexent -l app=nexent-elasticsearch Re-run the initialization script: ```bash -cd k8s/helm -bash init-elasticsearch.sh +bash deploy/k8s/init-elasticsearch.sh ``` ### Clean Up Stale PersistentVolumes @@ -364,5 +409,5 @@ bash init-elasticsearch.sh Released PVs are automatically cleaned during deployment. To manually clean: ```bash -kubectl delete pv nexent-elasticsearch-pv nexent-postgresql-pv nexent-redis-pv nexent-minio-pv +kubectl delete pv nexent-workspace-pv nexent-skills-pv nexent-elasticsearch-pv nexent-postgresql-pv nexent-redis-pv nexent-minio-pv ``` diff --git a/k8s/helm/nexent/charts/nexent-common/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-common/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-common/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-common/templates/configmap.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/templates/configmap.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-common/templates/configmap.yaml rename to deploy/k8s/helm/nexent/charts/nexent-common/templates/configmap.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-common/templates/init-sql-configmap.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/templates/init-sql-configmap.yaml new file mode 100644 index 000000000..da78ede39 --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-common/templates/init-sql-configmap.yaml @@ -0,0 +1,21 @@ +{{- $sqlFiles := default dict .Values.sqlFiles -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: nexent-sql-files + namespace: {{ .Values.global.namespace }} + annotations: + "helm.sh/hook-weight": "-3" +data: + init.sql: | +{{ default "" $sqlFiles.init | nindent 4 }} + migrations-.keep: "" +{{ range $name, $content := default dict $sqlFiles.migrations }} + {{ printf "migrations-%s" $name | quote }}: | +{{ $content | nindent 4 }} +{{ end }} + supabase-.keep: "" +{{ range $name, $content := default dict $sqlFiles.supabase }} + {{ printf "supabase-%s" $name | quote }}: | +{{ $content | nindent 4 }} +{{ end }} diff --git a/k8s/helm/nexent/charts/nexent-common/templates/rbac.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/templates/rbac.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-common/templates/rbac.yaml rename to deploy/k8s/helm/nexent/charts/nexent-common/templates/rbac.yaml diff --git a/k8s/helm/nexent/charts/nexent-common/templates/secrets.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/templates/secrets.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-common/templates/secrets.yaml rename to deploy/k8s/helm/nexent/charts/nexent-common/templates/secrets.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-common/templates/shared-storage.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/templates/shared-storage.yaml new file mode 100644 index 000000000..560dd8b45 --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-common/templates/shared-storage.yaml @@ -0,0 +1,98 @@ +{{- $global := default dict .Values.global }} +{{- $shared := default dict $global.sharedStorage }} +{{- $mode := default "local" $shared.mode }} +{{- $storageClassName := default "" $shared.storageClassName }} +{{- $accessModes := default (list "ReadWriteOnce") $shared.accessModes }} +{{- $workspace := default dict $shared.workspace }} +{{- $workspaceSize := default "10Gi" $workspace.size }} +{{- $workspaceLocalPath := default "/var/lib/nexent" $workspace.localPath }} +{{- if eq $mode "local" }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nexent-workspace-pv + labels: + type: hostpath + app: nexent-workspace + annotations: + "helm.sh/hook-weight": "-3" +spec: + storageClassName: {{ $storageClassName | quote }} + capacity: + storage: {{ $workspaceSize }} + accessModes: +{{ toYaml $accessModes | indent 4 }} + persistentVolumeReclaimPolicy: Retain + hostPath: + path: {{ $workspaceLocalPath | quote }} + type: DirectoryOrCreate +--- +{{- end }} +{{- if ne $mode "existing" }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nexent-workspace + namespace: {{ .Values.global.namespace }} + annotations: + "helm.sh/hook-weight": "-3" +spec: + accessModes: +{{ toYaml $accessModes | indent 4 }} + resources: + requests: + storage: {{ $workspaceSize }} + {{- if eq $mode "local" }} + volumeName: nexent-workspace-pv + {{- end }} + {{- if $storageClassName }} + storageClassName: {{ $storageClassName | quote }} + {{- end }} +--- +{{- end }} +{{- $skills := default dict $shared.skills }} +{{- $skillsSize := default "5Gi" $skills.size }} +{{- $skillsLocalPath := default "/var/lib/nexent-data/skills" $skills.localPath }} +{{- if eq $mode "local" }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nexent-skills-pv + labels: + type: hostpath + app: nexent-skills + annotations: + "helm.sh/hook-weight": "-3" +spec: + storageClassName: {{ $storageClassName | quote }} + capacity: + storage: {{ $skillsSize }} + accessModes: +{{ toYaml $accessModes | indent 4 }} + persistentVolumeReclaimPolicy: Retain + hostPath: + path: {{ $skillsLocalPath | quote }} + type: DirectoryOrCreate +--- +{{- end }} +{{- if ne $mode "existing" }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nexent-skills + namespace: {{ .Values.global.namespace }} + annotations: + "helm.sh/hook-weight": "-3" +spec: + accessModes: +{{ toYaml $accessModes | indent 4 }} + resources: + requests: + storage: {{ $skillsSize }} + {{- if eq $mode "local" }} + volumeName: nexent-skills-pv + {{- end }} + {{- if $storageClassName }} + storageClassName: {{ $storageClassName | quote }} + {{- end }} +{{- end }} diff --git a/k8s/helm/nexent/charts/nexent-common/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/values.yaml similarity index 95% rename from k8s/helm/nexent/charts/nexent-common/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-common/values.yaml index 7b27ba302..26bdafc22 100644 --- a/k8s/helm/nexent/charts/nexent-common/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-common/values.yaml @@ -1,5 +1,5 @@ # Nexent Common Chart - Shared resources configuration -# This chart provides shared resources (ConfigMap, Secret, RBAC, init.sql) +# This chart provides shared resources (ConfigMap, Secret, RBAC, SQL files) # that are required by other Nexent charts. # Images used by common templates @@ -9,6 +9,14 @@ images: tag: "latest" pullPolicy: IfNotPresent +# SQL content is rendered by deploy/k8s/deploy.sh from deploy/sql/ +# directory. Keep this empty in chart defaults to avoid maintaining a second SQL +# copy inside the chart. +sqlFiles: + init: "" + migrations: {} + supabase: {} + # ConfigMap data - this will be used by nexent-config ConfigMap config: # Service URLs (internal) @@ -43,7 +51,7 @@ config: skipProxy: "true" umask: "0022" isDeployedByKubernetes: "true" - skillsPath: "/mnt/nexent/skills" + skillsPath: "/mnt/nexent-data/skills" marketBackend: "http://60.204.251.153:8010" modelEngine: enabled: "false" @@ -189,19 +197,14 @@ secrets: storage: elasticsearch: size: "20Gi" - hostPath: "/var/lib/nexent-data/nexent-elasticsearch" postgresql: size: "10Gi" - hostPath: "/var/lib/nexent-data/nexent-postgresql" redis: size: "5Gi" - hostPath: "/var/lib/nexent-data/nexent-redis" minio: size: "20Gi" - hostPath: "/var/lib/nexent-data/nexent-minio" supabaseDb: size: "10Gi" - hostPath: "/var/lib/nexent-data/nexent-supabase-db" # Service account configuration serviceAccount: diff --git a/k8s/helm/nexent/charts/nexent-config/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-config/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-config/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-config/Chart.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-config/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-config/templates/deployment.yaml new file mode 100644 index 000000000..c31aa74bc --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-config/templates/deployment.yaml @@ -0,0 +1,93 @@ +{{- $global := default dict .Values.global -}} +{{- $sqlFileNames := default dict $global.sqlFileNames -}} +{{- $sharedStorage := default dict $global.sharedStorage -}} +{{- $workspaceStorage := default dict $sharedStorage.workspace -}} +{{- $skillsStorage := default dict $sharedStorage.skills -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nexent-config + namespace: {{ .Values.global.namespace }} + labels: + app: nexent-config + annotations: + "helm.sh/hook-weight": "20" +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: nexent-config + template: + metadata: + labels: + app: nexent-config + annotations: + checksum/nexent-backend: {{ dig "rolloutChecksums" "backend" "" .Values.global | quote }} + checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} + spec: + serviceAccountName: {{ .Values.serviceAccount.name }} + containers: + - name: nexent-config + image: "{{ .Values.images.backend.repository }}:{{ .Values.images.backend.tag }}" + imagePullPolicy: {{ .Values.images.backend.pullPolicy }} + ports: + - containerPort: 5010 + name: http + command: + - /opt/nexent/scripts/start-backend.sh + - python + - backend/config_service.py + envFrom: + - configMapRef: + name: nexent-config + - secretRef: + name: nexent-secrets + env: + - name: NEXENT_SQL_STARTUP_MODE + value: "migrate" + - name: DEPLOYMENT_VERSION + value: {{ .Values.global.deploymentVersion | quote }} + - name: skip_proxy + value: {{ .Values.config.skipProxy | quote }} + - name: UMASK + value: {{ .Values.config.umask | quote }} + volumeMounts: + - name: nexent-sql-files + mountPath: /opt/nexent/sql + readOnly: true + - name: nexent-workspace + mountPath: /mnt/nexent + - name: nexent-skills + mountPath: /mnt/nexent-data/skills + resources: + requests: + memory: {{ .Values.resources.backend.requests.memory }} + cpu: {{ .Values.resources.backend.requests.cpu }} + limits: + memory: {{ .Values.resources.backend.limits.memory }} + cpu: {{ .Values.resources.backend.limits.cpu }} + volumes: + - name: nexent-sql-files + configMap: + name: nexent-sql-files + items: + - key: init.sql + path: init.sql + - key: migrations-.keep + path: migrations/.keep +{{ range $name := default (list) $sqlFileNames.migrations }} + - key: {{ printf "migrations-%s" $name | quote }} + path: {{ printf "migrations/%s" $name | quote }} +{{ end }} + - key: supabase-.keep + path: supabase/.keep +{{ range $name := default (list) $sqlFileNames.supabase }} + - key: {{ printf "supabase-%s" $name | quote }} + path: {{ printf "supabase/%s" $name | quote }} +{{ end }} + - name: nexent-workspace + persistentVolumeClaim: + claimName: {{ default "nexent-workspace" $workspaceStorage.existingClaim }} + - name: nexent-skills + persistentVolumeClaim: + claimName: {{ default "nexent-skills" $skillsStorage.existingClaim }} diff --git a/k8s/helm/nexent/charts/nexent-config/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-config/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-config/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-config/templates/service.yaml diff --git a/k8s/helm/nexent/charts/nexent-config/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-config/values.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-config/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-config/values.yaml diff --git a/k8s/helm/nexent/charts/nexent-data-process/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-data-process/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-data-process/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-data-process/Chart.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-data-process/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-data-process/templates/deployment.yaml new file mode 100644 index 000000000..9637bd281 --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-data-process/templates/deployment.yaml @@ -0,0 +1,93 @@ +{{- $global := default dict .Values.global -}} +{{- $sqlFileNames := default dict $global.sqlFileNames -}} +{{- $sharedStorage := default dict $global.sharedStorage -}} +{{- $workspaceStorage := default dict $sharedStorage.workspace -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nexent-data-process + namespace: {{ .Values.global.namespace }} + labels: + app: nexent-data-process + annotations: + "helm.sh/hook-weight": "20" +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: nexent-data-process + template: + metadata: + labels: + app: nexent-data-process + annotations: + checksum/nexent-backend: {{ dig "rolloutChecksums" "backend" "" .Values.global | quote }} + checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} + spec: + containers: + - name: nexent-data-process + image: "{{ .Values.images.dataProcess.repository }}:{{ .Values.images.dataProcess.tag }}" + imagePullPolicy: {{ .Values.images.dataProcess.pullPolicy }} + ports: + - containerPort: 5012 + name: http + - containerPort: 5555 + name: flower + - containerPort: 8265 + name: ray-dashboard + command: + - /opt/nexent/scripts/start-backend.sh + - /bin/sh + - -c + - python /opt/backend/data_process_service.py || (cd /opt/backend && OPENBLAS_NUM_THREADS=1 UVICORN_LOOP=asyncio uvicorn data_process_service:app --host 0.0.0.0 --port 5012) + envFrom: + - configMapRef: + name: nexent-config + - secretRef: + name: nexent-secrets + env: + - name: NEXENT_SQL_STARTUP_MODE + value: "off" + - name: DEPLOYMENT_VERSION + value: {{ .Values.global.deploymentVersion | quote }} + - name: DOCKER_ENVIRONMENT + value: {{ .Values.config.dockerEnvironment | quote }} + - name: PYTHONPATH + value: {{ .Values.config.pythonPath | quote }} + - name: skip_proxy + value: {{ .Values.config.skipProxy | quote }} + volumeMounts: + - name: nexent-sql-files + mountPath: /opt/nexent/sql + readOnly: true + - name: nexent-workspace + mountPath: /mnt/nexent + resources: + requests: + memory: {{ .Values.resources.dataProcess.requests.memory }} + cpu: {{ .Values.resources.dataProcess.requests.cpu }} + limits: + memory: {{ .Values.resources.dataProcess.limits.memory }} + cpu: {{ .Values.resources.dataProcess.limits.cpu }} + volumes: + - name: nexent-sql-files + configMap: + name: nexent-sql-files + items: + - key: init.sql + path: init.sql + - key: migrations-.keep + path: migrations/.keep +{{ range $name := default (list) $sqlFileNames.migrations }} + - key: {{ printf "migrations-%s" $name | quote }} + path: {{ printf "migrations/%s" $name | quote }} +{{ end }} + - key: supabase-.keep + path: supabase/.keep +{{ range $name := default (list) $sqlFileNames.supabase }} + - key: {{ printf "supabase-%s" $name | quote }} + path: {{ printf "supabase/%s" $name | quote }} +{{ end }} + - name: nexent-workspace + persistentVolumeClaim: + claimName: {{ default "nexent-workspace" $workspaceStorage.existingClaim }} diff --git a/k8s/helm/nexent/charts/nexent-data-process/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-data-process/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-data-process/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-data-process/templates/service.yaml diff --git a/k8s/helm/nexent/charts/nexent-data-process/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-data-process/values.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-data-process/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-data-process/values.yaml diff --git a/k8s/helm/nexent/charts/nexent-elasticsearch/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-elasticsearch/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-elasticsearch/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-elasticsearch/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/templates/deployment.yaml similarity index 97% rename from k8s/helm/nexent/charts/nexent-elasticsearch/templates/deployment.yaml rename to deploy/k8s/helm/nexent/charts/nexent-elasticsearch/templates/deployment.yaml index 7bcc91f71..050527878 100644 --- a/k8s/helm/nexent/charts/nexent-elasticsearch/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/templates/deployment.yaml @@ -112,4 +112,4 @@ spec: volumes: - name: elasticsearch-data persistentVolumeClaim: - claimName: nexent-elasticsearch + claimName: {{ default "nexent-elasticsearch" .Values.persistence.existingClaim }} diff --git a/k8s/helm/nexent/charts/nexent-elasticsearch/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-elasticsearch/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-elasticsearch/templates/service.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/templates/storage.yaml b/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/templates/storage.yaml new file mode 100644 index 000000000..080a221c9 --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/templates/storage.yaml @@ -0,0 +1,44 @@ +{{- $mode := default "local" .Values.persistence.mode }} +{{- if eq $mode "local" }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nexent-elasticsearch-pv + labels: + type: hostpath + app: nexent-elasticsearch + annotations: + "helm.sh/hook-weight": "-3" +spec: + storageClassName: {{ .Values.persistence.storageClassName | quote }} + capacity: + storage: {{ .Values.storage.size }} + accessModes: +{{ toYaml .Values.persistence.accessModes | indent 4 }} + persistentVolumeReclaimPolicy: Retain + hostPath: + path: {{ .Values.persistence.localPath | quote }} + type: DirectoryOrCreate +--- +{{- end }} +{{- if ne $mode "existing" }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nexent-elasticsearch + namespace: {{ .Values.global.namespace }} + annotations: + "helm.sh/hook-weight": "-3" +spec: + accessModes: +{{ toYaml .Values.persistence.accessModes | indent 4 }} + resources: + requests: + storage: {{ .Values.storage.size }} + {{- if eq $mode "local" }} + volumeName: nexent-elasticsearch-pv + {{- end }} + {{- if .Values.persistence.storageClassName }} + storageClassName: {{ .Values.persistence.storageClassName | quote }} + {{- end }} +{{- end }} diff --git a/k8s/helm/nexent/charts/nexent-elasticsearch/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/values.yaml similarity index 67% rename from k8s/helm/nexent/charts/nexent-elasticsearch/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-elasticsearch/values.yaml index 8836214ac..620f7f7ad 100644 --- a/k8s/helm/nexent/charts/nexent-elasticsearch/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/values.yaml @@ -15,7 +15,14 @@ resources: storage: size: 20Gi - hostPath: "/var/lib/nexent-data/nexent-elasticsearch" + +persistence: + mode: local + storageClassName: nexent-local + accessModes: + - ReadWriteOnce + localPath: "/var/lib/nexent-data/nexent-elasticsearch" + existingClaim: "" config: javaOpts: "-Xms2g -Xmx2g" diff --git a/k8s/helm/nexent/charts/nexent-mcp/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-mcp/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-mcp/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-mcp/Chart.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-mcp/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-mcp/templates/deployment.yaml new file mode 100644 index 000000000..defa5f869 --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-mcp/templates/deployment.yaml @@ -0,0 +1,101 @@ +{{- $global := default dict .Values.global -}} +{{- $sqlFileNames := default dict $global.sqlFileNames -}} +{{- $sharedStorage := default dict $global.sharedStorage -}} +{{- $workspaceStorage := default dict $sharedStorage.workspace -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nexent-mcp + namespace: {{ .Values.global.namespace }} + labels: + app: nexent-mcp + annotations: + "helm.sh/hook-weight": "20" +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: nexent-mcp + template: + metadata: + labels: + app: nexent-mcp + annotations: + checksum/nexent-backend: {{ dig "rolloutChecksums" "backend" "" .Values.global | quote }} + checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} + spec: + containers: + - name: nexent-mcp + image: "{{ .Values.images.backend.repository }}:{{ .Values.images.backend.tag }}" + imagePullPolicy: {{ .Values.images.backend.pullPolicy }} + ports: + - containerPort: 5011 + name: http + - containerPort: 5015 + name: http-alt + command: + - /opt/nexent/scripts/start-backend.sh + - python + - backend/mcp_service.py + envFrom: + - configMapRef: + name: nexent-config + - secretRef: + name: nexent-secrets + env: + - name: NEXENT_SQL_STARTUP_MODE + value: "wait" + - name: DEPLOYMENT_VERSION + value: {{ .Values.global.deploymentVersion | quote }} + - name: skip_proxy + value: {{ .Values.config.skipProxy | quote }} + - name: UMASK + value: {{ .Values.config.umask | quote }} + volumeMounts: + - name: nexent-sql-files + mountPath: /opt/nexent/sql + readOnly: true + - name: nexent-workspace + mountPath: /mnt/nexent + resources: + requests: + memory: {{ .Values.resources.backend.requests.memory }} + cpu: {{ .Values.resources.backend.requests.cpu }} + limits: + memory: {{ .Values.resources.backend.limits.memory }} + cpu: {{ .Values.resources.backend.limits.cpu }} + readinessProbe: + tcpSocket: + port: 5011 + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + livenessProbe: + tcpSocket: + port: 5011 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 3 + volumes: + - name: nexent-sql-files + configMap: + name: nexent-sql-files + items: + - key: init.sql + path: init.sql + - key: migrations-.keep + path: migrations/.keep +{{ range $name := default (list) $sqlFileNames.migrations }} + - key: {{ printf "migrations-%s" $name | quote }} + path: {{ printf "migrations/%s" $name | quote }} +{{ end }} + - key: supabase-.keep + path: supabase/.keep +{{ range $name := default (list) $sqlFileNames.supabase }} + - key: {{ printf "supabase-%s" $name | quote }} + path: {{ printf "supabase/%s" $name | quote }} +{{ end }} + - name: nexent-workspace + persistentVolumeClaim: + claimName: {{ default "nexent-workspace" $workspaceStorage.existingClaim }} diff --git a/k8s/helm/nexent/charts/nexent-mcp/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-mcp/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-mcp/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-mcp/templates/service.yaml diff --git a/k8s/helm/nexent/charts/nexent-mcp/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-mcp/values.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-mcp/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-mcp/values.yaml diff --git a/k8s/helm/nexent/charts/nexent-minio/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-minio/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-minio/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-minio/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-minio/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-minio/templates/deployment.yaml similarity index 94% rename from k8s/helm/nexent/charts/nexent-minio/templates/deployment.yaml rename to deploy/k8s/helm/nexent/charts/nexent-minio/templates/deployment.yaml index 7467c8258..101cf726c 100644 --- a/k8s/helm/nexent/charts/nexent-minio/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-minio/templates/deployment.yaml @@ -16,6 +16,8 @@ spec: metadata: labels: app: nexent-minio + annotations: + checksum/nexent-minio: {{ dig "rolloutChecksums" "minio" "" .Values.global | quote }} spec: containers: - name: minio @@ -104,4 +106,4 @@ spec: volumes: - name: minio-data persistentVolumeClaim: - claimName: nexent-minio + claimName: {{ default "nexent-minio" .Values.persistence.existingClaim }} diff --git a/k8s/helm/nexent/charts/nexent-minio/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-minio/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-minio/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-minio/templates/service.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-minio/templates/storage.yaml b/deploy/k8s/helm/nexent/charts/nexent-minio/templates/storage.yaml new file mode 100644 index 000000000..21a48d6df --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-minio/templates/storage.yaml @@ -0,0 +1,44 @@ +{{- $mode := default "local" .Values.persistence.mode }} +{{- if eq $mode "local" }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nexent-minio-pv + labels: + type: hostpath + app: nexent-minio + annotations: + "helm.sh/hook-weight": "-3" +spec: + storageClassName: {{ .Values.persistence.storageClassName | quote }} + capacity: + storage: {{ .Values.storage.size }} + accessModes: +{{ toYaml .Values.persistence.accessModes | indent 4 }} + persistentVolumeReclaimPolicy: Retain + hostPath: + path: {{ .Values.persistence.localPath | quote }} + type: DirectoryOrCreate +--- +{{- end }} +{{- if ne $mode "existing" }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nexent-minio + namespace: {{ .Values.global.namespace }} + annotations: + "helm.sh/hook-weight": "-3" +spec: + accessModes: +{{ toYaml .Values.persistence.accessModes | indent 4 }} + resources: + requests: + storage: {{ .Values.storage.size }} + {{- if eq $mode "local" }} + volumeName: nexent-minio-pv + {{- end }} + {{- if .Values.persistence.storageClassName }} + storageClassName: {{ .Values.persistence.storageClassName | quote }} + {{- end }} +{{- end }} diff --git a/k8s/helm/nexent/charts/nexent-minio/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-minio/values.yaml similarity index 66% rename from k8s/helm/nexent/charts/nexent-minio/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-minio/values.yaml index 784d50588..a8ee99381 100644 --- a/k8s/helm/nexent/charts/nexent-minio/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-minio/values.yaml @@ -15,7 +15,14 @@ resources: storage: size: 20Gi - hostPath: "/var/lib/nexent-data/nexent-minio" + +persistence: + mode: local + storageClassName: nexent-local + accessModes: + - ReadWriteOnce + localPath: "/var/lib/nexent-data/nexent-minio" + existingClaim: "" service: type: ClusterIP diff --git a/k8s/helm/nexent/charts/nexent-monitoring/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-monitoring/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-monitoring/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-monitoring/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-monitoring/templates/_helpers.tpl b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/_helpers.tpl similarity index 77% rename from k8s/helm/nexent/charts/nexent-monitoring/templates/_helpers.tpl rename to deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/_helpers.tpl index e466a3d7b..dd7c0fa26 100644 --- a/k8s/helm/nexent/charts/nexent-monitoring/templates/_helpers.tpl +++ b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/_helpers.tpl @@ -39,6 +39,65 @@ {{- if or .Values.langfuse.enabled (eq (include "nexent-monitoring.provider" .) "langfuse") -}}true{{- end -}} {{- end -}} +{{- define "nexent-monitoring.claimName" -}} +{{- $root := .root -}} +{{- $name := .name -}} +{{- $mode := default "local" $root.Values.persistence.mode -}} +{{- $prefix := default "" $root.Values.persistence.existingClaimPrefix -}} +{{- if and (eq $mode "existing") $prefix -}}{{ printf "%s-%s" $prefix $name }}{{- else -}}{{ $name }}{{- end -}} +{{- end -}} + +{{- define "nexent-monitoring.persistentStorage" -}} +{{- $root := .root -}} +{{- $name := .name -}} +{{- $size := .size -}} +{{- $mode := default "local" $root.Values.persistence.mode -}} +{{- $storageClassName := default "" $root.Values.persistence.storageClassName -}} +{{- $localPath := default "/var/lib/nexent-data" $root.Values.persistence.localPath -}} +{{- $accessModes := default (list "ReadWriteOnce") $root.Values.persistence.accessModes -}} +{{- if and $root.Values.enabled $root.Values.persistence.enabled -}} +{{- if eq $mode "local" }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ printf "%s-pv" $name }} + labels: + app: {{ $name }} +spec: + storageClassName: {{ $storageClassName | quote }} + capacity: + storage: {{ $size }} + accessModes: +{{ toYaml $accessModes | indent 4 }} + persistentVolumeReclaimPolicy: Retain + hostPath: + path: {{ printf "%s/%s" $localPath $name | quote }} + type: DirectoryOrCreate +--- +{{- end }} +{{- if ne $mode "existing" }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ $name }} + namespace: {{ $root.Values.global.namespace }} +spec: + accessModes: +{{ toYaml $accessModes | indent 4 }} + resources: + requests: + storage: {{ $size }} + {{- if eq $mode "local" }} + volumeName: {{ printf "%s-pv" $name }} + {{- end }} + {{- if $storageClassName }} + storageClassName: {{ $storageClassName | quote }} + {{- end }} +--- +{{- end }} +{{- end -}} +{{- end -}} + {{- define "nexent-monitoring.langfuseAuthHeader" -}} {{- if .Values.collector.env.langfuseOtlpAuthHeader -}} {{- .Values.collector.env.langfuseOtlpAuthHeader -}} diff --git a/k8s/helm/nexent/charts/nexent-monitoring/templates/grafana-tempo.yaml b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/grafana-tempo.yaml similarity index 97% rename from k8s/helm/nexent/charts/nexent-monitoring/templates/grafana-tempo.yaml rename to deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/grafana-tempo.yaml index ca8ce5f26..64953f851 100644 --- a/k8s/helm/nexent/charts/nexent-monitoring/templates/grafana-tempo.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/grafana-tempo.yaml @@ -90,7 +90,7 @@ spec: - name: tempo-data {{- if .Values.persistence.enabled }} persistentVolumeClaim: - claimName: nexent-tempo + claimName: {{ include "nexent-monitoring.claimName" (dict "root" . "name" "nexent-tempo") }} {{- else }} emptyDir: {} {{- end }} @@ -240,7 +240,7 @@ spec: - name: grafana-data {{- if .Values.persistence.enabled }} persistentVolumeClaim: - claimName: nexent-grafana + claimName: {{ include "nexent-monitoring.claimName" (dict "root" . "name" "nexent-grafana") }} {{- else }} emptyDir: {} {{- end }} diff --git a/k8s/helm/nexent/charts/nexent-monitoring/templates/langfuse.yaml b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/langfuse.yaml similarity index 95% rename from k8s/helm/nexent/charts/nexent-monitoring/templates/langfuse.yaml rename to deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/langfuse.yaml index ba2ecb33b..6646b8ae5 100644 --- a/k8s/helm/nexent/charts/nexent-monitoring/templates/langfuse.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/langfuse.yaml @@ -41,7 +41,7 @@ spec: - name: postgres-data {{- if .Values.persistence.enabled }} persistentVolumeClaim: - claimName: nexent-langfuse-postgres + claimName: {{ include "nexent-monitoring.claimName" (dict "root" . "name" "nexent-langfuse-postgres") }} {{- else }} emptyDir: {} {{- end }} @@ -105,7 +105,7 @@ spec: - name: clickhouse-data {{- if .Values.persistence.enabled }} persistentVolumeClaim: - claimName: nexent-langfuse-clickhouse + claimName: {{ include "nexent-monitoring.claimName" (dict "root" . "name" "nexent-langfuse-clickhouse") }} {{- else }} emptyDir: {} {{- end }} @@ -171,7 +171,7 @@ spec: - name: minio-data {{- if .Values.persistence.enabled }} persistentVolumeClaim: - claimName: nexent-langfuse-minio + claimName: {{ include "nexent-monitoring.claimName" (dict "root" . "name" "nexent-langfuse-minio") }} {{- else }} emptyDir: {} {{- end }} @@ -231,7 +231,7 @@ spec: - name: redis-data {{- if .Values.persistence.enabled }} persistentVolumeClaim: - claimName: nexent-langfuse-redis + claimName: {{ include "nexent-monitoring.claimName" (dict "root" . "name" "nexent-langfuse-redis") }} {{- else }} emptyDir: {} {{- end }} diff --git a/k8s/helm/nexent/charts/nexent-monitoring/templates/otel-collector-configmap.yaml b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/otel-collector-configmap.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-monitoring/templates/otel-collector-configmap.yaml rename to deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/otel-collector-configmap.yaml diff --git a/k8s/helm/nexent/charts/nexent-monitoring/templates/otel-collector.yaml b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/otel-collector.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-monitoring/templates/otel-collector.yaml rename to deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/otel-collector.yaml diff --git a/k8s/helm/nexent/charts/nexent-monitoring/templates/phoenix.yaml b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/phoenix.yaml similarity index 94% rename from k8s/helm/nexent/charts/nexent-monitoring/templates/phoenix.yaml rename to deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/phoenix.yaml index d22f9c3f5..4620de184 100644 --- a/k8s/helm/nexent/charts/nexent-monitoring/templates/phoenix.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/phoenix.yaml @@ -35,7 +35,7 @@ spec: - name: phoenix-data {{- if .Values.persistence.enabled }} persistentVolumeClaim: - claimName: nexent-phoenix + claimName: {{ include "nexent-monitoring.claimName" (dict "root" . "name" "nexent-phoenix") }} {{- else }} emptyDir: {} {{- end }} diff --git a/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/storage.yaml b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/storage.yaml new file mode 100644 index 000000000..27becfd63 --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/storage.yaml @@ -0,0 +1,15 @@ +{{- if include "nexent-monitoring.phoenixEnabled" . }} +{{ include "nexent-monitoring.persistentStorage" (dict "root" . "name" "nexent-phoenix" "size" .Values.phoenix.storage.size) }} +{{- end }} +{{- if include "nexent-monitoring.tempoEnabled" . }} +{{ include "nexent-monitoring.persistentStorage" (dict "root" . "name" "nexent-tempo" "size" .Values.tempo.storage.size) }} +{{- end }} +{{- if include "nexent-monitoring.grafanaEnabled" . }} +{{ include "nexent-monitoring.persistentStorage" (dict "root" . "name" "nexent-grafana" "size" .Values.grafana.storage.size) }} +{{- end }} +{{- if include "nexent-monitoring.langfuseEnabled" . }} +{{ include "nexent-monitoring.persistentStorage" (dict "root" . "name" "nexent-langfuse-postgres" "size" .Values.langfuse.postgres.storage.size) }} +{{ include "nexent-monitoring.persistentStorage" (dict "root" . "name" "nexent-langfuse-clickhouse" "size" .Values.langfuse.clickhouse.storage.dataSize) }} +{{ include "nexent-monitoring.persistentStorage" (dict "root" . "name" "nexent-langfuse-minio" "size" .Values.langfuse.minio.storage.size) }} +{{ include "nexent-monitoring.persistentStorage" (dict "root" . "name" "nexent-langfuse-redis" "size" .Values.langfuse.redis.storage.size) }} +{{- end }} diff --git a/k8s/helm/nexent/charts/nexent-monitoring/templates/zipkin.yaml b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/zipkin.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-monitoring/templates/zipkin.yaml rename to deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/zipkin.yaml diff --git a/k8s/helm/nexent/charts/nexent-monitoring/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-monitoring/values.yaml similarity index 86% rename from k8s/helm/nexent/charts/nexent-monitoring/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-monitoring/values.yaml index 7be3c03ff..76cf76862 100644 --- a/k8s/helm/nexent/charts/nexent-monitoring/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-monitoring/values.yaml @@ -83,7 +83,6 @@ phoenix: grpcPort: 4317 storage: size: 10Gi - hostPath: /var/lib/nexent-data/nexent-phoenix grafana: enabled: false @@ -96,7 +95,6 @@ grafana: nodePort: 30002 storage: size: 5Gi - hostPath: /var/lib/nexent-data/nexent-grafana tempo: enabled: false @@ -107,7 +105,6 @@ tempo: otlpHttpPort: 4318 storage: size: 10Gi - hostPath: /var/lib/nexent-data/nexent-tempo zipkin: enabled: false @@ -144,29 +141,28 @@ langfuse: database: postgres storage: size: 10Gi - hostPath: /var/lib/nexent-data/nexent-langfuse-postgres clickhouse: user: clickhouse password: clickhouse storage: dataSize: 20Gi - dataHostPath: /var/lib/nexent-data/nexent-langfuse-clickhouse logSize: 5Gi - logHostPath: /var/lib/nexent-data/nexent-langfuse-clickhouse-logs minio: rootUser: minio rootPassword: miniosecret bucket: langfuse storage: size: 10Gi - hostPath: /var/lib/nexent-data/nexent-langfuse-minio redis: auth: myredissecret storage: size: 5Gi - hostPath: /var/lib/nexent-data/nexent-langfuse-redis persistence: enabled: true - createPv: true - storageClassName: hostpath + mode: local + storageClassName: nexent-local + accessModes: + - ReadWriteOnce + localPath: /var/lib/nexent-data + existingClaimPrefix: "" diff --git a/k8s/helm/nexent/charts/nexent-northbound/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-northbound/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-northbound/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-northbound/Chart.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml new file mode 100644 index 000000000..d2a49039e --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml @@ -0,0 +1,92 @@ +{{- $global := default dict .Values.global -}} +{{- $sqlFileNames := default dict $global.sqlFileNames -}} +{{- $sharedStorage := default dict $global.sharedStorage -}} +{{- $workspaceStorage := default dict $sharedStorage.workspace -}} +{{- $skillsStorage := default dict $sharedStorage.skills -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nexent-northbound + namespace: {{ .Values.global.namespace }} + labels: + app: nexent-northbound + annotations: + "helm.sh/hook-weight": "20" +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: nexent-northbound + template: + metadata: + labels: + app: nexent-northbound + annotations: + checksum/nexent-backend: {{ dig "rolloutChecksums" "backend" "" .Values.global | quote }} + checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} + spec: + containers: + - name: nexent-northbound + image: "{{ .Values.images.backend.repository }}:{{ .Values.images.backend.tag }}" + imagePullPolicy: {{ .Values.images.backend.pullPolicy }} + ports: + - containerPort: 5013 + name: http + command: + - /opt/nexent/scripts/start-backend.sh + - python + - backend/northbound_service.py + envFrom: + - configMapRef: + name: nexent-config + - secretRef: + name: nexent-secrets + env: + - name: NEXENT_SQL_STARTUP_MODE + value: "wait" + - name: DEPLOYMENT_VERSION + value: {{ .Values.global.deploymentVersion | quote }} + - name: skip_proxy + value: {{ .Values.config.skipProxy | quote }} + - name: UMASK + value: {{ .Values.config.umask | quote }} + volumeMounts: + - name: nexent-sql-files + mountPath: /opt/nexent/sql + readOnly: true + - name: nexent-workspace + mountPath: /mnt/nexent + - name: nexent-skills + mountPath: /mnt/nexent-data/skills + resources: + requests: + memory: {{ .Values.resources.backend.requests.memory }} + cpu: {{ .Values.resources.backend.requests.cpu }} + limits: + memory: {{ .Values.resources.backend.limits.memory }} + cpu: {{ .Values.resources.backend.limits.cpu }} + volumes: + - name: nexent-sql-files + configMap: + name: nexent-sql-files + items: + - key: init.sql + path: init.sql + - key: migrations-.keep + path: migrations/.keep +{{ range $name := default (list) $sqlFileNames.migrations }} + - key: {{ printf "migrations-%s" $name | quote }} + path: {{ printf "migrations/%s" $name | quote }} +{{ end }} + - key: supabase-.keep + path: supabase/.keep +{{ range $name := default (list) $sqlFileNames.supabase }} + - key: {{ printf "supabase-%s" $name | quote }} + path: {{ printf "supabase/%s" $name | quote }} +{{ end }} + - name: nexent-workspace + persistentVolumeClaim: + claimName: {{ default "nexent-workspace" $workspaceStorage.existingClaim }} + - name: nexent-skills + persistentVolumeClaim: + claimName: {{ default "nexent-skills" $skillsStorage.existingClaim }} diff --git a/k8s/helm/nexent/charts/nexent-northbound/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-northbound/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-northbound/templates/service.yaml diff --git a/k8s/helm/nexent/charts/nexent-northbound/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-northbound/values.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-northbound/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-northbound/values.yaml diff --git a/k8s/helm/nexent/charts/nexent-openssh/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-openssh/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-openssh/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-openssh/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-openssh/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-openssh/templates/deployment.yaml similarity index 92% rename from k8s/helm/nexent/charts/nexent-openssh/templates/deployment.yaml rename to deploy/k8s/helm/nexent/charts/nexent-openssh/templates/deployment.yaml index 713b8d348..4921c832d 100644 --- a/k8s/helm/nexent/charts/nexent-openssh/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-openssh/templates/deployment.yaml @@ -17,6 +17,8 @@ spec: metadata: labels: app: nexent-openssh-server + annotations: + checksum/nexent-ssh: {{ dig "rolloutChecksums" "ssh" "" .Values.global | quote }} spec: containers: - name: openssh-server diff --git a/k8s/helm/nexent/charts/nexent-openssh/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-openssh/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-openssh/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-openssh/templates/service.yaml diff --git a/k8s/helm/nexent/charts/nexent-openssh/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-openssh/values.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-openssh/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-openssh/values.yaml diff --git a/k8s/helm/nexent/charts/nexent-postgresql/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-postgresql/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-postgresql/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-postgresql/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-postgresql/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-postgresql/templates/deployment.yaml similarity index 84% rename from k8s/helm/nexent/charts/nexent-postgresql/templates/deployment.yaml rename to deploy/k8s/helm/nexent/charts/nexent-postgresql/templates/deployment.yaml index bd7df8b0f..0f4cc0c8e 100644 --- a/k8s/helm/nexent/charts/nexent-postgresql/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-postgresql/templates/deployment.yaml @@ -16,6 +16,8 @@ spec: metadata: labels: app: nexent-postgresql + annotations: + checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} spec: containers: @@ -38,7 +40,7 @@ spec: volumeMounts: - name: postgresql-data mountPath: /var/lib/postgresql/data - - name: init-sql + - name: nexent-sql-files mountPath: /docker-entrypoint-initdb.d/init.sql subPath: init.sql resources: @@ -53,7 +55,7 @@ spec: volumes: - name: postgresql-data persistentVolumeClaim: - claimName: nexent-postgresql - - name: init-sql + claimName: {{ default "nexent-postgresql" .Values.persistence.existingClaim }} + - name: nexent-sql-files configMap: - name: nexent-init-sql + name: nexent-sql-files diff --git a/k8s/helm/nexent/charts/nexent-postgresql/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-postgresql/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-postgresql/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-postgresql/templates/service.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-postgresql/templates/storage.yaml b/deploy/k8s/helm/nexent/charts/nexent-postgresql/templates/storage.yaml new file mode 100644 index 000000000..914f75de4 --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-postgresql/templates/storage.yaml @@ -0,0 +1,44 @@ +{{- $mode := default "local" .Values.persistence.mode }} +{{- if eq $mode "local" }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nexent-postgresql-pv + labels: + type: hostpath + app: nexent-postgresql + annotations: + "helm.sh/hook-weight": "-3" +spec: + storageClassName: {{ .Values.persistence.storageClassName | quote }} + capacity: + storage: {{ .Values.storage.size }} + accessModes: +{{ toYaml .Values.persistence.accessModes | indent 4 }} + persistentVolumeReclaimPolicy: Retain + hostPath: + path: {{ .Values.persistence.localPath | quote }} + type: DirectoryOrCreate +--- +{{- end }} +{{- if ne $mode "existing" }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nexent-postgresql + namespace: {{ .Values.global.namespace }} + annotations: + "helm.sh/hook-weight": "-3" +spec: + accessModes: +{{ toYaml .Values.persistence.accessModes | indent 4 }} + resources: + requests: + storage: {{ .Values.storage.size }} + {{- if eq $mode "local" }} + volumeName: nexent-postgresql-pv + {{- end }} + {{- if .Values.persistence.storageClassName }} + storageClassName: {{ .Values.persistence.storageClassName | quote }} + {{- end }} +{{- end }} diff --git a/k8s/helm/nexent/charts/nexent-postgresql/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-postgresql/values.yaml similarity index 62% rename from k8s/helm/nexent/charts/nexent-postgresql/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-postgresql/values.yaml index 52eced034..eeb6b2e38 100644 --- a/k8s/helm/nexent/charts/nexent-postgresql/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-postgresql/values.yaml @@ -15,7 +15,14 @@ resources: storage: size: 10Gi - hostPath: "/var/lib/nexent-data/nexent-postgresql" + +persistence: + mode: local + storageClassName: nexent-local + accessModes: + - ReadWriteOnce + localPath: "/var/lib/nexent-data/nexent-postgresql" + existingClaim: "" config: host: "nexent-postgresql" diff --git a/k8s/helm/nexent/charts/nexent-redis/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-redis/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-redis/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-redis/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-redis/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-redis/templates/deployment.yaml similarity index 95% rename from k8s/helm/nexent/charts/nexent-redis/templates/deployment.yaml rename to deploy/k8s/helm/nexent/charts/nexent-redis/templates/deployment.yaml index f33388edd..426ba9a5c 100644 --- a/k8s/helm/nexent/charts/nexent-redis/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-redis/templates/deployment.yaml @@ -68,4 +68,4 @@ spec: volumes: - name: redis-data persistentVolumeClaim: - claimName: nexent-redis + claimName: {{ default "nexent-redis" .Values.persistence.existingClaim }} diff --git a/k8s/helm/nexent/charts/nexent-redis/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-redis/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-redis/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-redis/templates/service.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-redis/templates/storage.yaml b/deploy/k8s/helm/nexent/charts/nexent-redis/templates/storage.yaml new file mode 100644 index 000000000..02ed5a67b --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-redis/templates/storage.yaml @@ -0,0 +1,44 @@ +{{- $mode := default "local" .Values.persistence.mode }} +{{- if eq $mode "local" }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nexent-redis-pv + labels: + type: hostpath + app: nexent-redis + annotations: + "helm.sh/hook-weight": "-3" +spec: + storageClassName: {{ .Values.persistence.storageClassName | quote }} + capacity: + storage: {{ .Values.storage.size }} + accessModes: +{{ toYaml .Values.persistence.accessModes | indent 4 }} + persistentVolumeReclaimPolicy: Retain + hostPath: + path: {{ .Values.persistence.localPath | quote }} + type: DirectoryOrCreate +--- +{{- end }} +{{- if ne $mode "existing" }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nexent-redis + namespace: {{ .Values.global.namespace }} + annotations: + "helm.sh/hook-weight": "-3" +spec: + accessModes: +{{ toYaml .Values.persistence.accessModes | indent 4 }} + resources: + requests: + storage: {{ .Values.storage.size }} + {{- if eq $mode "local" }} + volumeName: nexent-redis-pv + {{- end }} + {{- if .Values.persistence.storageClassName }} + storageClassName: {{ .Values.persistence.storageClassName | quote }} + {{- end }} +{{- end }} diff --git a/k8s/helm/nexent/charts/nexent-redis/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-redis/values.yaml similarity index 55% rename from k8s/helm/nexent/charts/nexent-redis/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-redis/values.yaml index e24c7adc5..3c94070b4 100644 --- a/k8s/helm/nexent/charts/nexent-redis/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-redis/values.yaml @@ -15,4 +15,11 @@ resources: storage: size: 5Gi - hostPath: "/var/lib/nexent-data/nexent-redis" + +persistence: + mode: local + storageClassName: nexent-local + accessModes: + - ReadWriteOnce + localPath: "/var/lib/nexent-data/nexent-redis" + existingClaim: "" diff --git a/k8s/helm/nexent/charts/nexent-runtime/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-runtime/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-runtime/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-runtime/Chart.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-runtime/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-runtime/templates/deployment.yaml new file mode 100644 index 000000000..411d04500 --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-runtime/templates/deployment.yaml @@ -0,0 +1,92 @@ +{{- $global := default dict .Values.global -}} +{{- $sqlFileNames := default dict $global.sqlFileNames -}} +{{- $sharedStorage := default dict $global.sharedStorage -}} +{{- $workspaceStorage := default dict $sharedStorage.workspace -}} +{{- $skillsStorage := default dict $sharedStorage.skills -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nexent-runtime + namespace: {{ .Values.global.namespace }} + labels: + app: nexent-runtime + annotations: + "helm.sh/hook-weight": "20" +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: nexent-runtime + template: + metadata: + labels: + app: nexent-runtime + annotations: + checksum/nexent-backend: {{ dig "rolloutChecksums" "backend" "" .Values.global | quote }} + checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} + spec: + containers: + - name: nexent-runtime + image: "{{ .Values.images.backend.repository }}:{{ .Values.images.backend.tag }}" + imagePullPolicy: {{ .Values.images.backend.pullPolicy }} + ports: + - containerPort: 5014 + name: http + command: + - /opt/nexent/scripts/start-backend.sh + - python + - backend/runtime_service.py + envFrom: + - configMapRef: + name: nexent-config + - secretRef: + name: nexent-secrets + env: + - name: NEXENT_SQL_STARTUP_MODE + value: "wait" + - name: DEPLOYMENT_VERSION + value: {{ .Values.global.deploymentVersion | quote }} + - name: skip_proxy + value: {{ .Values.config.skipProxy | quote }} + - name: UMASK + value: {{ .Values.config.umask | quote }} + volumeMounts: + - name: nexent-sql-files + mountPath: /opt/nexent/sql + readOnly: true + - name: nexent-workspace + mountPath: /mnt/nexent + - name: nexent-skills + mountPath: /mnt/nexent-data/skills + resources: + requests: + memory: {{ .Values.resources.backend.requests.memory }} + cpu: {{ .Values.resources.backend.requests.cpu }} + limits: + memory: {{ .Values.resources.backend.limits.memory }} + cpu: {{ .Values.resources.backend.limits.cpu }} + volumes: + - name: nexent-sql-files + configMap: + name: nexent-sql-files + items: + - key: init.sql + path: init.sql + - key: migrations-.keep + path: migrations/.keep +{{ range $name := default (list) $sqlFileNames.migrations }} + - key: {{ printf "migrations-%s" $name | quote }} + path: {{ printf "migrations/%s" $name | quote }} +{{ end }} + - key: supabase-.keep + path: supabase/.keep +{{ range $name := default (list) $sqlFileNames.supabase }} + - key: {{ printf "supabase-%s" $name | quote }} + path: {{ printf "supabase/%s" $name | quote }} +{{ end }} + - name: nexent-workspace + persistentVolumeClaim: + claimName: {{ default "nexent-workspace" $workspaceStorage.existingClaim }} + - name: nexent-skills + persistentVolumeClaim: + claimName: {{ default "nexent-skills" $skillsStorage.existingClaim }} diff --git a/k8s/helm/nexent/charts/nexent-runtime/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-runtime/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-runtime/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-runtime/templates/service.yaml diff --git a/k8s/helm/nexent/charts/nexent-runtime/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-runtime/values.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-runtime/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-runtime/values.yaml diff --git a/k8s/helm/nexent/charts/nexent-supabase-auth/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-supabase-auth/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-auth/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-supabase-auth/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/templates/deployment.yaml similarity index 97% rename from k8s/helm/nexent/charts/nexent-supabase-auth/templates/deployment.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-auth/templates/deployment.yaml index ea75b639e..46ec3c137 100644 --- a/k8s/helm/nexent/charts/nexent-supabase-auth/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/templates/deployment.yaml @@ -18,6 +18,8 @@ spec: metadata: labels: app: nexent-supabase-auth + annotations: + checksum/nexent-supabase: {{ dig "rolloutChecksums" "supabase" "" .Values.global | quote }} spec: initContainers: - name: init-db diff --git a/k8s/helm/nexent/charts/nexent-supabase-auth/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-supabase-auth/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-auth/templates/service.yaml diff --git a/k8s/helm/nexent/charts/nexent-supabase-auth/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/values.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-supabase-auth/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-auth/values.yaml diff --git a/k8s/helm/nexent/charts/nexent-supabase-db/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-supabase-db/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-db/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-supabase-db/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/deployment.yaml similarity index 70% rename from k8s/helm/nexent/charts/nexent-supabase-db/templates/deployment.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/deployment.yaml index 55ed5f437..2d8f7acfc 100644 --- a/k8s/helm/nexent/charts/nexent-supabase-db/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/deployment.yaml @@ -1,4 +1,6 @@ {{- if or (eq .Values.global.deploymentVersion "full") (index .Values.global.deploymentComponents "supabase") }} +{{- $global := default dict .Values.global -}} +{{- $sqlFileNames := default dict $global.sqlFileNames -}} --- apiVersion: apps/v1 kind: Deployment @@ -18,6 +20,9 @@ spec: metadata: labels: app: nexent-supabase-db + annotations: + checksum/nexent-supabase: {{ dig "rolloutChecksums" "supabase" "" .Values.global | quote }} + checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} spec: initContainers: - name: init-db @@ -28,25 +33,22 @@ spec: - | echo "Copying init scripts into existing image script directory..." cp -r /docker-entrypoint-initdb.d/* /initdb.d/ - cp /custom-init-scripts/98-webhooks.sql /initdb.d/init-scripts/ - cp /custom-init-scripts/99-roles.sql /initdb.d/init-scripts/ - cp /custom-init-scripts/99-jwt.sql /initdb.d/init-scripts/ + cp /custom-supabase-sql/webhooks.sql /initdb.d/init-scripts/98-webhooks.sql + cp /custom-supabase-sql/roles.sql /initdb.d/init-scripts/99-roles.sql + cp /custom-supabase-sql/jwt.sql /initdb.d/init-scripts/99-jwt.sql - cp /custom-init-scripts/99-logs.sql /initdb.d/migrations/ - cp /custom-init-scripts/99-realtime.sql /initdb.d/migrations/ - cp /custom-init-scripts/97-_supabase.sql /initdb.d/migrations/ - cp /custom-init-scripts/99-pooler.sql /initdb.d/migrations/ + cp /custom-supabase-sql/logs.sql /initdb.d/migrations/99-logs.sql + cp /custom-supabase-sql/realtime.sql /initdb.d/migrations/99-realtime.sql + cp /custom-supabase-sql/_supabase.sql /initdb.d/migrations/97-_supabase.sql + cp /custom-supabase-sql/pooler.sql /initdb.d/migrations/99-pooler.sql - echo "Copying user-defined migration scripts..." - cp /custom-migrations/* /initdb.d/migrations/ || echo "Skip migrations" echo "Initialization scripts are ready" volumeMounts: - - mountPath: /custom-init-scripts - name: custom-init-scripts + - mountPath: /custom-supabase-sql + name: custom-supabase-sql + readOnly: true - mountPath: /initdb.d name: initdb-scripts-data - - mountPath: /custom-migrations - name: custom-migrations containers: - name: supabase-db image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -115,13 +117,17 @@ spec: - name: initdb-scripts-data emptyDir: medium: "" - - name: custom-init-scripts + - name: custom-supabase-sql configMap: - name: nexent-supabase-db-init - - name: custom-migrations - configMap: - name: nexent-supabase-db-migrations + name: nexent-sql-files + items: + - key: supabase-.keep + path: .keep +{{ range $name := default (list) $sqlFileNames.supabase }} + - key: {{ printf "supabase-%s" $name | quote }} + path: {{ $name | quote }} +{{ end }} - name: supabase-db-data persistentVolumeClaim: - claimName: nexent-supabase-db + claimName: {{ default "nexent-supabase-db" .Values.persistence.existingClaim }} {{- end }} diff --git a/k8s/helm/nexent/charts/nexent-supabase-db/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-supabase-db/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/service.yaml diff --git a/deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/storage.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/storage.yaml new file mode 100644 index 000000000..5c2f9d265 --- /dev/null +++ b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/storage.yaml @@ -0,0 +1,47 @@ +{{- if or (eq .Values.global.deploymentVersion "full") (index .Values.global.deploymentComponents "supabase") }} +{{- $mode := default "local" .Values.persistence.mode }} +{{- if eq $mode "local" }} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nexent-supabase-db-pv + labels: + type: hostpath + app: nexent-supabase-db + annotations: + "helm.sh/hook-weight": "-2" +spec: + storageClassName: {{ .Values.persistence.storageClassName | quote }} + capacity: + storage: {{ .Values.storage.size }} + accessModes: +{{ toYaml .Values.persistence.accessModes | indent 4 }} + persistentVolumeReclaimPolicy: Retain + hostPath: + path: {{ .Values.persistence.localPath | quote }} + type: DirectoryOrCreate +{{- end }} +{{- if ne $mode "existing" }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nexent-supabase-db + namespace: {{ .Values.global.namespace }} + annotations: + "helm.sh/hook-weight": "-2" +spec: + accessModes: +{{ toYaml .Values.persistence.accessModes | indent 4 }} + resources: + requests: + storage: {{ .Values.storage.size }} + {{- if eq $mode "local" }} + volumeName: nexent-supabase-db-pv + {{- end }} + {{- if .Values.persistence.storageClassName }} + storageClassName: {{ .Values.persistence.storageClassName | quote }} + {{- end }} +{{- end }} +{{- end }} diff --git a/k8s/helm/nexent/charts/nexent-supabase-db/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/values.yaml similarity index 63% rename from k8s/helm/nexent/charts/nexent-supabase-db/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-db/values.yaml index fb93a58af..fc61e6c93 100644 --- a/k8s/helm/nexent/charts/nexent-supabase-db/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/values.yaml @@ -15,7 +15,14 @@ resources: storage: size: 10Gi - hostPath: "/var/lib/nexent-data/nexent-supabase-db" + +persistence: + mode: local + storageClassName: nexent-local + accessModes: + - ReadWriteOnce + localPath: "/var/lib/nexent-data/nexent-supabase-db" + existingClaim: "" config: postgresDb: "supabase" diff --git a/k8s/helm/nexent/charts/nexent-supabase-kong/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-kong/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-supabase-kong/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-kong/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-supabase-kong/templates/configmap.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/configmap.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-supabase-kong/templates/configmap.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/configmap.yaml diff --git a/k8s/helm/nexent/charts/nexent-supabase-kong/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/deployment.yaml similarity index 96% rename from k8s/helm/nexent/charts/nexent-supabase-kong/templates/deployment.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/deployment.yaml index 584d41eac..296b74656 100644 --- a/k8s/helm/nexent/charts/nexent-supabase-kong/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/deployment.yaml @@ -18,6 +18,8 @@ spec: metadata: labels: app: nexent-supabase-kong + annotations: + checksum/nexent-supabase: {{ dig "rolloutChecksums" "supabase" "" .Values.global | quote }} spec: containers: - name: kong diff --git a/k8s/helm/nexent/charts/nexent-supabase-kong/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-supabase-kong/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/service.yaml diff --git a/k8s/helm/nexent/charts/nexent-supabase-kong/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-kong/values.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-supabase-kong/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-supabase-kong/values.yaml diff --git a/k8s/helm/nexent/charts/nexent-web/Chart.yaml b/deploy/k8s/helm/nexent/charts/nexent-web/Chart.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-web/Chart.yaml rename to deploy/k8s/helm/nexent/charts/nexent-web/Chart.yaml diff --git a/k8s/helm/nexent/charts/nexent-web/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-web/templates/deployment.yaml similarity index 89% rename from k8s/helm/nexent/charts/nexent-web/templates/deployment.yaml rename to deploy/k8s/helm/nexent/charts/nexent-web/templates/deployment.yaml index e13547a80..729fdfbd0 100644 --- a/k8s/helm/nexent/charts/nexent-web/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-web/templates/deployment.yaml @@ -16,6 +16,8 @@ spec: metadata: labels: app: nexent-web + annotations: + checksum/nexent-web: {{ dig "rolloutChecksums" "web" "" .Values.global | quote }} spec: containers: - name: nexent-web @@ -35,6 +37,8 @@ spec: value: "http://nexent-runtime:5014" - name: MINIO_ENDPOINT value: "http://nexent-minio:9000" + - name: DEPLOYMENT_VERSION + value: {{ .Values.global.deploymentVersion | quote }} - name: MARKET_BACKEND value: {{ .Values.config.marketBackend | quote }} - name: MODEL_ENGINE_ENABLED diff --git a/k8s/helm/nexent/charts/nexent-web/templates/service.yaml b/deploy/k8s/helm/nexent/charts/nexent-web/templates/service.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-web/templates/service.yaml rename to deploy/k8s/helm/nexent/charts/nexent-web/templates/service.yaml diff --git a/k8s/helm/nexent/charts/nexent-web/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-web/values.yaml similarity index 100% rename from k8s/helm/nexent/charts/nexent-web/values.yaml rename to deploy/k8s/helm/nexent/charts/nexent-web/values.yaml diff --git a/k8s/helm/nexent/templates/_helpers.tpl b/deploy/k8s/helm/nexent/templates/_helpers.tpl similarity index 100% rename from k8s/helm/nexent/templates/_helpers.tpl rename to deploy/k8s/helm/nexent/templates/_helpers.tpl diff --git a/k8s/helm/nexent/templates/ingress.yaml b/deploy/k8s/helm/nexent/templates/ingress.yaml similarity index 100% rename from k8s/helm/nexent/templates/ingress.yaml rename to deploy/k8s/helm/nexent/templates/ingress.yaml diff --git a/k8s/helm/nexent/values.yaml b/deploy/k8s/helm/nexent/values.yaml similarity index 85% rename from k8s/helm/nexent/values.yaml rename to deploy/k8s/helm/nexent/values.yaml index 6224d0949..bda678f7b 100644 --- a/k8s/helm/nexent/values.yaml +++ b/deploy/k8s/helm/nexent/values.yaml @@ -2,12 +2,25 @@ global: namespace: nexent dataDir: "/var/lib/nexent-data" - deploymentVersion: "speed" + sharedStorage: + mode: "local" + storageClassName: "nexent-local" + accessModes: + - ReadWriteOnce + workspace: + size: "10Gi" + localPath: "/var/lib/nexent" + existingClaim: "nexent-workspace" + skills: + size: "5Gi" + localPath: "/var/lib/nexent-data/skills" + existingClaim: "nexent-skills" + deploymentVersion: "full" deploymentComponents: infrastructure: true application: true - data-process: false - supabase: false + data-process: true + supabase: true terminal: false monitoring: false portPolicy: "development" @@ -86,13 +99,13 @@ nexent-northbound: nexent-web: enabled: true nexent-data-process: - enabled: false + enabled: true nexent-supabase-kong: - enabled: false + enabled: true nexent-supabase-auth: - enabled: false + enabled: true nexent-supabase-db: - enabled: false + enabled: true nexent-openssh: enabled: false nexent-monitoring: diff --git a/deploy/k8s/init-elasticsearch.sh b/deploy/k8s/init-elasticsearch.sh new file mode 100644 index 000000000..d43450491 --- /dev/null +++ b/deploy/k8s/init-elasticsearch.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# Script to initialize Elasticsearch API key for Nexent + +NAMESPACE=nexent +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +ROOT_ENV_FILE="${ROOT_ENV_FILE:-$PROJECT_ROOT/.env}" +DEPLOYMENT_COMMON="$DEPLOY_ROOT/common/common.sh" + +if [ -f "$DEPLOYMENT_COMMON" ]; then + # shellcheck source=/dev/null + source "$DEPLOYMENT_COMMON" +fi + +decode_base64() { + if base64 --help 2>&1 | grep -q -- '--decode'; then + base64 --decode + else + base64 -D + fi +} + +get_secret_value() { + local key="$1" + local encoded_value + encoded_value=$(kubectl get secret nexent-secrets -n $NAMESPACE -o jsonpath="{.data.${key}}" 2>/dev/null || true) + [ -n "$encoded_value" ] || return 1 + printf '%s' "$encoded_value" | decode_base64 +} + +validate_api_key() { + local api_key="$1" + local http_code + [ -n "$api_key" ] || return 1 + http_code=$(kubectl exec -n $NAMESPACE deploy/nexent-elasticsearch -- sh -c "curl -s -o /dev/null -w '%{http_code}' -H 'Authorization: ApiKey $api_key' 'http://localhost:9200/_security/_authenticate'" 2>/dev/null || true) + [ "$http_code" = "200" ] +} +write_api_key_output() { + local api_key="$1" + if [ -n "${ELASTICSEARCH_API_KEY_OUTPUT_FILE:-}" ]; then + umask 077 + printf '%s' "$api_key" > "$ELASTICSEARCH_API_KEY_OUTPUT_FILE" + else + echo "ELASTICSEARCH_API_KEY=$api_key" + fi +} + +sync_api_key_to_root_env() { + local api_key="$1" + + if [ "${NEXENT_SYNC_ES_KEY_TO_ENV:-true}" != "true" ]; then + return 0 + fi + + if command -v deployment_update_env_var_file >/dev/null 2>&1; then + deployment_update_env_var_file "$ROOT_ENV_FILE" "ELASTICSEARCH_API_KEY" "$api_key" + else + touch "$ROOT_ENV_FILE" + local escaped_value + escaped_value=$(printf '%s' "$api_key" | sed -e 's/\\/\\\\/g' -e 's/&/\\&/g') + if grep -q '^ELASTICSEARCH_API_KEY=' "$ROOT_ENV_FILE"; then + sed -i.bak "s~^ELASTICSEARCH_API_KEY=.*~ELASTICSEARCH_API_KEY=\"${escaped_value}\"~" "$ROOT_ENV_FILE" + rm -f "${ROOT_ENV_FILE}.bak" + else + printf 'ELASTICSEARCH_API_KEY="%s"\n' "$api_key" >> "$ROOT_ENV_FILE" + fi + fi + + echo "ELASTICSEARCH_API_KEY synchronized to $ROOT_ENV_FILE." +} + +# Get elastic password from secret +ELASTIC_PASSWORD=$(get_secret_value "ELASTIC_PASSWORD") + +echo "Waiting for Elasticsearch to be ready..." + +# Wait for Elasticsearch to be healthy +until kubectl exec -n $NAMESPACE deploy/nexent-elasticsearch -- curl -s -u "elastic:$ELASTIC_PASSWORD" "http://localhost:9200/_cluster/health" 2>/dev/null | grep -q '"status":"green"\|"status":"yellow"'; do + echo "Elasticsearch is unavailable - sleeping" + sleep 5 +done +echo "Elasticsearch is ready." + +EXISTING_API_KEY="$(get_secret_value "ELASTICSEARCH_API_KEY" 2>/dev/null || true)" +if [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" != "true" ] && [ "${DEPLOYMENT_REFRESH_ES_KEY:-false}" != "true" ] && [ -n "$EXISTING_API_KEY" ]; then + echo "Validating existing ELASTICSEARCH_API_KEY..." + if validate_api_key "$EXISTING_API_KEY"; then + echo "Existing ELASTICSEARCH_API_KEY is valid; keeping current Helm-managed value." + write_api_key_output "$EXISTING_API_KEY" + exit 0 + fi + echo "Existing ELASTICSEARCH_API_KEY is invalid; generating a replacement." +elif [ "${DEPLOYMENT_ROTATE_SECRETS:-false}" = "true" ] || [ "${DEPLOYMENT_REFRESH_ES_KEY:-false}" = "true" ]; then + echo "ELASTICSEARCH_API_KEY refresh requested; generating a replacement." +fi + +echo "Generating API key..." + +# Generate API key +API_KEY_JSON=$(kubectl exec -n $NAMESPACE deploy/nexent-elasticsearch -- sh -c "curl -s -u 'elastic:$ELASTIC_PASSWORD' 'http://localhost:9200/_security/api_key' -H 'Content-Type: application/json' -d '{\"name\":\"nexent_api_key\",\"role_descriptors\":{\"nexent_role\":{\"cluster\":[\"all\"],\"index\":[{\"names\":[\"*\"],\"privileges\":[\"all\"]}]}}}'") + +echo "API Key Response: $API_KEY_JSON" + +# Extract API key using sed instead of jq +ENCODED_KEY=$(echo "$API_KEY_JSON" | sed 's/.*"encoded":"\([^"]*\)".*/\1/') + +echo "Extracted key: $ENCODED_KEY" + +if [ -n "$ENCODED_KEY" ] && [ "$ENCODED_KEY" != "$API_KEY_JSON" ]; then + echo "Generated ELASTICSEARCH_API_KEY: $ENCODED_KEY" + + write_api_key_output "$ENCODED_KEY" + sync_api_key_to_root_env "$ENCODED_KEY" + echo "ELASTICSEARCH_API_KEY generated; Helm will update nexent-secrets." +else + echo "Failed to extract API key from response" + echo "Full response: $API_KEY_JSON" + exit 1 +fi diff --git a/k8s/helm/uninstall.sh b/deploy/k8s/uninstall.sh similarity index 82% rename from k8s/helm/uninstall.sh rename to deploy/k8s/uninstall.sh index d902fe784..1ee6f249a 100755 --- a/k8s/helm/uninstall.sh +++ b/deploy/k8s/uninstall.sh @@ -34,7 +34,7 @@ print_usage() { echo " --delete-volumes true|false Alias for --delete-data" echo " --remove-volumes Alias for --delete-data true" echo " --keep-volumes Alias for --delete-data false" - echo " --delete-local-data true|false Control whether hostPath data is deleted" + echo " --delete-local-data true|false Control whether local PV data is deleted" echo " --remove-local-data Alias for --delete-local-data true" echo " --keep-local-data Alias for --delete-local-data false" echo " --delete-namespace true|false Control whether the namespace is deleted" @@ -159,6 +159,23 @@ clean_helm_state() { echo "Helm state cleaned." } +helm_uninstall_release() { + local output + if output=$(helm uninstall "$RELEASE_NAME" --namespace "$NAMESPACE" 2>&1); then + [ -z "$output" ] || printf '%s\n' "$output" + return 0 + fi + + local status=$? + [ -z "$output" ] || printf '%s\n' "$output" + if printf '%s\n' "$output" | grep -qi 'not found'; then + echo "Helm release '$RELEASE_NAME' is already absent; continuing cleanup." + return 0 + fi + + return "$status" +} + delete_namespace_after_uninstall() { echo "Deleting namespace..." kubectl delete namespace "$NAMESPACE" --ignore-not-found=true || true @@ -190,6 +207,8 @@ maybe_delete_namespace_after_uninstall() { local_volume_paths() { printf '%s\n' \ + "/var/lib/nexent" \ + "/var/lib/nexent-data/skills" \ "/var/lib/nexent-data/nexent-elasticsearch" \ "/var/lib/nexent-data/nexent-postgresql" \ "/var/lib/nexent-data/nexent-redis" \ @@ -214,7 +233,7 @@ resolve_delete_local_data() { [ -t 0 ] || return 1 echo "" - echo "Delete local hostPath volume data under /var/lib/nexent-data?" + echo "Delete local PV data under /var/lib/nexent and /var/lib/nexent-data?" local answer read -r -p "Delete local volume data? [y/N]: " answer answer="$(sanitize_input "$answer")" @@ -222,12 +241,12 @@ resolve_delete_local_data() { } delete_local_volume_data() { - echo "Deleting local hostPath volume data..." + echo "Deleting local PV data..." local path while IFS= read -r path; do case "$path" in - /var/lib/nexent-data/nexent-*) + /var/lib/nexent|/var/lib/nexent-data/skills|/var/lib/nexent-data/nexent-*) if [ -e "$path" ]; then echo "Removing $path" rm -rf -- "$path" @@ -246,13 +265,27 @@ maybe_delete_local_volume_data() { if resolve_delete_local_data; then delete_local_volume_data else - echo "Local hostPath volume data preserved." + echo "Local PV data preserved." fi } +cleanup_leftover_data_process_resources() { + if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + return 0 + fi + + echo "Cleaning up leftover nexent-data-process resources..." + kubectl delete deployment nexent-data-process -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true + kubectl delete service nexent-data-process -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true + kubectl delete rs,pod -n "$NAMESPACE" -l app=nexent-data-process --ignore-not-found=true 2>/dev/null || true +} + uninstall_preserve_data() { echo "Uninstalling Helm release..." - helm uninstall "$RELEASE_NAME" --namespace "$NAMESPACE" + if ! helm_uninstall_release; then + echo "Helm uninstall failed; continuing best-effort cleanup of nexent-data-process." + fi + cleanup_leftover_data_process_resources maybe_delete_local_volume_data maybe_delete_namespace_after_uninstall echo "Cleanup completed. Helm-managed resources were removed." @@ -265,10 +298,12 @@ uninstall_preserve_data() { delete_all_data() { echo "Deleting Helm release..." - if ! helm uninstall "$RELEASE_NAME" --namespace "$NAMESPACE"; then + if ! helm_uninstall_release; then echo "Helm uninstall failed. Namespace was not deleted." + cleanup_leftover_data_process_resources return 1 fi + cleanup_leftover_data_process_resources maybe_delete_local_volume_data maybe_delete_namespace_after_uninstall echo "Cleanup completed. Helm-managed PV/PVC resources were deleted with the release." diff --git a/deploy/offline/build_offline_package.sh b/deploy/offline/build_offline_package.sh new file mode 100755 index 000000000..1c27251de --- /dev/null +++ b/deploy/offline/build_offline_package.sh @@ -0,0 +1,693 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +DEPLOY_ROOT="$PROJECT_ROOT/deploy" +DEPLOYMENT_COMMON="$DEPLOY_ROOT/common/common.sh" +VERSION_HELPER="$DEPLOY_ROOT/common/version.sh" + +DEFAULT_VERSION="latest" +DEFAULT_PLATFORM="amd64" +DEFAULT_OUTPUT_DIR="$PROJECT_ROOT/offline-package" +DEFAULT_INCLUDE_SOURCE="false" +DEFAULT_TARGET="all" +DEFAULT_COMPRESS="false" + +VERSION="" +PLATFORM="" +OUTPUT_DIR="" +INCLUDE_SOURCE="" +TARGET="" +COMPRESS="" +DRY_RUN="false" +COMMON_ARGS=() + +if [ -f "$DEPLOYMENT_COMMON" ]; then + # shellcheck source=/dev/null + source "$DEPLOYMENT_COMMON" +else + echo "Error: shared deployment helper not found: $DEPLOYMENT_COMMON" + exit 1 +fi + +if [ -f "$VERSION_HELPER" ]; then + # shellcheck source=/dev/null + source "$VERSION_HELPER" +fi + +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Build offline deployment package for Nexent" + echo "" + echo "Options:" + echo " --version VERSION Nexent image version (e.g. v1.0.0 or latest)" + echo " Default: $DEFAULT_VERSION" + echo " --platform PLATFORM Target platform (amd64 or arm64)" + echo " Default: $DEFAULT_PLATFORM" + echo " --output-dir DIR Output directory for the package" + echo " Default: $DEFAULT_OUTPUT_DIR" + echo " --include-source BOOL Include source code (true or false)" + echo " Default: $DEFAULT_INCLUDE_SOURCE" + echo " --target TARGET docker, k8s, or all" + echo " Default: $DEFAULT_TARGET" + echo " --compress BOOL Create zip archive after package build (true or false)" + echo " Default: $DEFAULT_COMPRESS" + echo " --components LIST Deployment components for image selection" + echo " --image-source SOURCE general, mainland, or local-latest" + echo " --registry-profile NAME Legacy alias for --image-source general|mainland" + echo " --config FILE Deployment config with components and image source" + echo " --dry-run Show execution plan without actual operations" + echo " --help Show this help message" + echo "" + echo "Examples:" + echo " $0 --version v1.0.0 --platform arm64" + echo " $0 --version latest --platform amd64 --include-source false" + echo " $0 --dry-run # Show execution plan without actual operations" +} + +parse_args() { + local dry_run=false + + while [[ $# -gt 0 ]]; do + case "$1" in + --version) + VERSION="$2" + shift 2 + ;; + --platform) + PLATFORM="$2" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --include-source) + INCLUDE_SOURCE="$2" + shift 2 + ;; + --target) + TARGET="$2" + shift 2 + ;; + --compress) + COMPRESS="$2" + shift 2 + ;; + --dry-run) + DRY_RUN="true" + shift + ;; + --components|--image-source|--registry-profile|--app-version|--monitoring-provider|--port-policy|--config|--local-config) + COMMON_ARGS+=("$1" "$2") + shift 2 + ;; + --use-local-config|--reconfigure) + COMMON_ARGS+=("$1") + shift + ;; + --help) + show_help + exit 0 + ;; + *) + echo "Unknown option: $1" + show_help + exit 1 + ;; + esac + done + + if declare -F deployment_read_version >/dev/null 2>&1; then + VERSION="${VERSION:-$(deployment_read_version "")}" + else + VERSION="${VERSION:-$DEFAULT_VERSION}" + fi + PLATFORM="${PLATFORM:-$DEFAULT_PLATFORM}" + OUTPUT_DIR="${OUTPUT_DIR:-$DEFAULT_OUTPUT_DIR}" + INCLUDE_SOURCE="${INCLUDE_SOURCE:-$DEFAULT_INCLUDE_SOURCE}" + TARGET="${TARGET:-$DEFAULT_TARGET}" + COMPRESS="${COMPRESS:-$DEFAULT_COMPRESS}" + + if [[ "$PLATFORM" != "amd64" && "$PLATFORM" != "arm64" ]]; then + echo "Error: Platform must be 'amd64' or 'arm64'" + exit 1 + fi + if [[ "$TARGET" != "docker" && "$TARGET" != "k8s" && "$TARGET" != "all" ]]; then + echo "Error: Target must be 'docker', 'k8s', or 'all'" + exit 1 + fi + if [[ "$COMPRESS" != "true" && "$COMPRESS" != "false" ]]; then + echo "Error: Compress must be 'true' or 'false'" + exit 1 + fi +} + +prepare_deployment_image_config() { + export APP_VERSION="$VERSION" + deployment_prepare_config "${COMMON_ARGS[@]}" --app-version "$VERSION" || exit 1 + + case "$DEPLOYMENT_REGISTRY_PROFILE" in + mainland) + [ -f "$DEPLOY_ROOT/env/image-source.mainland.env" ] && source "$DEPLOY_ROOT/env/image-source.mainland.env" + ;; + general|local-latest) + [ -f "$DEPLOY_ROOT/env/image-source.general.env" ] && source "$DEPLOY_ROOT/env/image-source.general.env" + ;; + esac + + deployment_apply_image_source +} + +show_dry_run_plan() { + echo "=== DRY RUN MODE ===" + echo "Version: $VERSION" + echo "Platform: $PLATFORM" + echo "Output directory: $OUTPUT_DIR" + echo "Include source: $INCLUDE_SOURCE" + echo "Target: $TARGET" + echo "Compress: $COMPRESS" + echo "Components: $DEPLOYMENT_COMPONENTS" + echo "Image source: $DEPLOYMENT_IMAGE_SOURCE" + echo "" + echo "Images to pull:" + get_nexent_images + get_third_party_images + echo "" + echo "No actual operations will be performed." + exit 0 +} + +get_nexent_images() { + deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "application" && echo "$NEXENT_IMAGE" + deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "application" && echo "$NEXENT_WEB_IMAGE" + deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "application" && echo "$NEXENT_MCP_DOCKER_IMAGE" + deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "data-process" && echo "$NEXENT_DATA_PROCESS_IMAGE" + deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "terminal" && echo "$OPENSSH_SERVER_IMAGE" + true +} + +get_third_party_images() { + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "infrastructure"; then + echo "$ELASTICSEARCH_IMAGE" + echo "$POSTGRESQL_IMAGE" + echo "$REDIS_IMAGE" + echo "$MINIO_IMAGE" + fi + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "supabase"; then + echo "$SUPABASE_KONG" + echo "$SUPABASE_GOTRUE" + echo "$SUPABASE_DB" + fi + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring"; then + echo "otel/opentelemetry-collector-contrib:0.151.0" + case "$DEPLOYMENT_MONITORING_PROVIDER" in + phoenix) echo "arizephoenix/phoenix:15" ;; + grafana) + echo "grafana/tempo:2.10.5" + echo "grafana/grafana:12.4" + ;; + zipkin) echo "openzipkin/zipkin:latest" ;; + langfuse) + echo "docker.io/langfuse/langfuse-worker:3" + echo "docker.io/langfuse/langfuse:3" + echo "docker.io/clickhouse/clickhouse-server:26.3-alpine" + echo "docker.io/minio/minio:RELEASE.2023-12-20T01-00-02Z" + echo "docker.io/redis:alpine" + echo "docker.io/postgres:15-alpine" + ;; + esac + fi + true +} + +uses_latest_tag() { + local image="$1" + local tag="${image##*:}" + [[ "$tag" == "latest" ]] +} + +image_exists_locally() { + local image="$1" + docker image inspect "$image" >/dev/null 2>&1 +} + +should_skip_pull() { + local image="$1" + + if image_exists_locally "$image"; then + echo "Using existing local image without pulling: $image" + return 0 + fi + + if uses_latest_tag "$image"; then + echo "Skipping pull for latest image; expecting local image: $image" + return 0 + fi + + return 1 +} + +pull_with_retry() { + local image="$1" + local platform="$2" + local max_retries=3 + local retry=0 + local wait_time=5 + + echo "Pulling image: $image (platform: $platform)" + + while [[ $retry -lt $max_retries ]]; do + if docker pull --platform "linux/$platform" "$image"; then + echo "✅ Successfully pulled: $image" + return 0 + fi + + retry=$((retry + 1)) + echo "⚠️ Pull failed (attempt $retry/$max_retries), retrying in $wait_time seconds..." + sleep $wait_time + done + + echo "❌ Failed to pull image after $max_retries attempts: $image" + return 1 +} + +pull_all_images() { + echo "" + echo "========================================" + echo "Pulling Nexent images..." + echo "========================================" + + local nexent_images_str + nexent_images_str=$(get_nexent_images) + + while IFS= read -r image; do + if should_skip_pull "$image"; then + continue + fi + + pull_with_retry "$image" "$PLATFORM" || { + echo "❌ Failed to pull Nexent image: $image" + return 1 + } + done <<< "$nexent_images_str" + + echo "" + echo "========================================" + echo "Pulling third-party images..." + echo "========================================" + + local third_party_images_str + third_party_images_str=$(get_third_party_images) + + while IFS= read -r image; do + if should_skip_pull "$image"; then + continue + fi + + pull_with_retry "$image" "$PLATFORM" || { + echo "❌ Failed to pull third-party image: $image" + return 1 + } + done <<< "$third_party_images_str" + + echo "" + echo "✅ All images pulled successfully" +} + +save_image_to_tar() { + local image="$1" + local output_file="$2" + + echo "Saving image to tar: $output_file" + + if docker save -o "$output_file" "$image"; then + echo "✅ Saved: $output_file" + return 0 + else + echo "❌ Failed to save image: $image" + return 1 + fi +} + +save_all_images() { + local images_dir="$OUTPUT_DIR/images" + + mkdir -p "$images_dir" + + echo "" + echo "========================================" + echo "Saving images to tar files..." + echo "========================================" + + local nexent_images_str + nexent_images_str=$(get_nexent_images) + + while IFS= read -r image; do + local image_name + image_name=$(echo "$image" | sed 's/.*\///' | sed 's/:.*//') + local image_tag + image_tag=$(echo "$image" | sed 's/.*://' | sed 's/\./-/g') + local tar_file="$images_dir/${image_name}-${image_tag}.tar" + + save_image_to_tar "$image" "$tar_file" || return 1 + done <<< "$nexent_images_str" + + local third_party_images_str + third_party_images_str=$(get_third_party_images) + + while IFS= read -r image; do + local image_name + image_name=$(echo "$image" | sed 's/.*\///' | sed 's/:.*//') + local image_tag + image_tag=$(echo "$image" | sed 's/.*://' | sed 's/RELEASE\.//' | sed 's/\./-/g') + local tar_file="$images_dir/${image_name}-${image_tag}.tar" + + save_image_to_tar "$image" "$tar_file" || return 1 + done <<< "$third_party_images_str" + + echo "" + echo "✅ All images saved successfully" +} + +copy_source_code() { + if [[ "$INCLUDE_SOURCE" != "true" ]]; then + echo "Skipping source code copy (include-source=false)" + return 0 + fi + + local source_dir="$OUTPUT_DIR/nexent" + + echo "" + echo "========================================" + echo "Copying git-managed source code..." + echo "========================================" + + echo "Source: $PROJECT_ROOT" + echo "Destination: $source_dir" + + rm -rf "$source_dir" + + mkdir -p "$source_dir" + + if ! git -C "$PROJECT_ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "⚠️ Warning: Project root is not a git repository" + echo " Falling back to copying all files (excluding .git and .github)" + + local cp_result=0 + if command -v rsync >/dev/null 2>&1; then + rsync -a --exclude='.git' --exclude='.github' "$PROJECT_ROOT/" "$source_dir/" || cp_result=$? + else + shopt -s dotglob nullglob + cp -r "$PROJECT_ROOT"/* "$source_dir/" 2>&1 || cp_result=$? + shopt -u dotglob nullglob + rm -rf "$source_dir/.git" "$source_dir/.github" + fi + + if [[ $cp_result -ne 0 ]]; then + echo "❌ Failed to copy source code" + return 1 + fi + + echo "✅ Source code copied to: $source_dir" + return 0 + fi + + echo " Using git ls-files to get managed file list..." + + local git_files + git_files=$(git -C "$PROJECT_ROOT" ls-files) + + if [[ -z "$git_files" ]]; then + echo "❌ No git-managed files found" + return 1 + fi + + local file_count + file_count=$(echo "$git_files" | wc -l | tr -d ' ') + echo " Found $file_count git-managed files" + + local file + while IFS= read -r file; do + local src_file="$PROJECT_ROOT/$file" + local dst_file="$source_dir/$file" + local dst_dir + + dst_dir=$(dirname "$dst_file") + + if [[ -f "$src_file" ]]; then + mkdir -p "$dst_dir" + cp "$src_file" "$dst_file" + fi + done <<< "$git_files" + + echo "✅ Git-managed source code copied to: $source_dir" + + local total_size + total_size=$(du -sh "$source_dir" | cut -f1) + echo " Total size: $total_size" + + return 0 +} + +create_load_script() { + local load_script="$OUTPUT_DIR/load-images.sh" + + echo "" + echo "========================================" + echo "Creating load-images.sh script..." + echo "========================================" + + cat > "$load_script" << 'LOADSCRIPT' +#!/bin/bash + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IMAGES_DIR="$SCRIPT_DIR/images" + +echo "Loading Docker images from $IMAGES_DIR..." + +for tar_file in "$IMAGES_DIR"/*.tar; do + if [[ -f "$tar_file" ]]; then + echo "Loading: $tar_file" + docker load -i "$tar_file" + fi +done + +echo "" +echo "✅ All images loaded successfully" +LOADSCRIPT + + chmod +x "$load_script" + + echo "✅ Created: $load_script" +} + +copy_deployment_bundle() { + echo "" + echo "========================================" + echo "Copying deployment bundle..." + echo "========================================" + + cp "$PROJECT_ROOT/deploy.sh" "$OUTPUT_DIR/deploy.sh" + cp "$PROJECT_ROOT/uninstall.sh" "$OUTPUT_DIR/uninstall.sh" + cp "$PROJECT_ROOT/VERSION" "$OUTPUT_DIR/VERSION" + cp "$PROJECT_ROOT/.env.example" "$OUTPUT_DIR/.env.example" + + if command -v rsync >/dev/null 2>&1; then + rsync -a \ + --exclude='.DS_Store' \ + --exclude='deploy.options' \ + --exclude='docker/.env.generated' \ + --exclude='k8s/helm/nexent/generated-values.yaml' \ + --exclude='k8s/helm/nexent/generated-runtime-values.yaml' \ + --exclude='k8s/helm/nexent/generated-secrets-values.yaml' \ + --exclude='k8s/helm/nexent/generated-persistence-values.yaml' \ + "$DEPLOY_ROOT/" "$OUTPUT_DIR/deploy/" + else + cp -R "$DEPLOY_ROOT" "$OUTPUT_DIR/deploy" + find "$OUTPUT_DIR" -name '.DS_Store' -type f -delete 2>/dev/null || true + fi + + rm -f "$OUTPUT_DIR/deploy/docker/.env.generated" "$OUTPUT_DIR/deploy/docker/deploy.options" "$OUTPUT_DIR/deploy/k8s/deploy.options" + rm -f "$OUTPUT_DIR/deploy/k8s/helm/nexent/generated-values.yaml" "$OUTPUT_DIR/deploy/k8s/helm/nexent/generated-runtime-values.yaml" "$OUTPUT_DIR/deploy/k8s/helm/nexent/generated-secrets-values.yaml" "$OUTPUT_DIR/deploy/k8s/helm/nexent/generated-persistence-values.yaml" + case "$TARGET" in + docker) rm -rf "$OUTPUT_DIR/deploy/k8s" ;; + k8s) rm -rf "$OUTPUT_DIR/deploy/docker" ;; + esac + + find "$OUTPUT_DIR" -name '.git' -type d -prune -exec rm -rf {} + 2>/dev/null || true + chmod +x "$OUTPUT_DIR/deploy.sh" "$OUTPUT_DIR/uninstall.sh" "$OUTPUT_DIR/load-images.sh" 2>/dev/null || true + find "$OUTPUT_DIR/deploy" -type f -name '*.sh' -exec chmod +x {} \; 2>/dev/null || true + + echo "✅ Deployment bundle copied" +} + +create_manifest() { + local manifest="$OUTPUT_DIR/manifest.yaml" + local image + + echo "" + echo "========================================" + echo "Creating manifest.yaml..." + echo "========================================" + + { + echo "version: \"$VERSION\"" + echo "platform: \"$PLATFORM\"" + echo "target: \"$TARGET\"" + echo "components: \"$DEPLOYMENT_COMPONENTS\"" + echo "imageSource: \"$DEPLOYMENT_IMAGE_SOURCE\"" + echo "images:" + while IFS= read -r image; do + [ -n "$image" ] && echo " - \"$image\"" + done < <(get_nexent_images; get_third_party_images) + } > "$manifest" + + echo "✅ Created: $manifest" +} + +create_checksums() { + local checksum_file="$OUTPUT_DIR/checksums.txt" + echo "" + echo "========================================" + echo "Creating checksums.txt..." + echo "========================================" + + if command -v sha256sum >/dev/null 2>&1; then + ( + cd "$OUTPUT_DIR" + find . -type f ! -name checksums.txt -print | LC_ALL=C sort | while IFS= read -r file; do + sha256sum "$file" + done + ) > "$checksum_file" + elif command -v shasum >/dev/null 2>&1; then + ( + cd "$OUTPUT_DIR" + find . -type f ! -name checksums.txt -print | LC_ALL=C sort | while IFS= read -r file; do + shasum -a 256 "$file" + done + ) > "$checksum_file" + else + echo "❌ sha256sum or shasum is required to create checksums" + return 1 + fi + + echo "✅ Created: $checksum_file" +} + +offline_package_name() { + local safe_version="${VERSION//\//-}" + echo "nexent-offline-${TARGET}-${PLATFORM}-${safe_version}" +} + +create_zip_package() { + if [[ "$COMPRESS" != "true" ]]; then + echo "Skipping zip archive creation (compress=false)" + return 0 + fi + + if ! command -v zip >/dev/null 2>&1; then + echo "❌ zip is required to create compressed package" + return 1 + fi + + local output_parent + local archive_file + + output_parent="$(cd "$(dirname "$OUTPUT_DIR")" && pwd)" + archive_file="$output_parent/$(offline_package_name).zip" + + echo "" + echo "========================================" + echo "Creating zip package..." + echo "========================================" + + rm -f "$archive_file" + (cd "$OUTPUT_DIR" && zip -r "$archive_file" .) + + echo "✅ Created: $archive_file" + ls -lh "$archive_file" +} + +main() { + parse_args "$@" + prepare_deployment_image_config + + if [[ "$DRY_RUN" == "true" ]]; then + show_dry_run_plan + fi + + echo "" + echo "========================================" + echo "Building Offline Deployment Package" + echo "========================================" + echo "Version: $VERSION" + echo "Platform: $PLATFORM" + echo "Output directory: $OUTPUT_DIR" + echo "Include source: $INCLUDE_SOURCE" + echo "Target: $TARGET" + echo "Compress: $COMPRESS" + echo "Components: $DEPLOYMENT_COMPONENTS" + echo "Image source: $DEPLOYMENT_IMAGE_SOURCE" + echo "========================================" + + rm -rf "$OUTPUT_DIR" + mkdir -p "$OUTPUT_DIR" + + pull_all_images || { + echo "❌ Image pull failed, aborting" + exit 1 + } + + save_all_images || { + echo "❌ Image save failed, aborting" + exit 1 + } + + copy_source_code || { + echo "❌ Source code copy failed, aborting" + exit 1 + } + + create_load_script || { + echo "❌ Load script creation failed, aborting" + exit 1 + } + + copy_deployment_bundle || { + echo "❌ Deployment bundle copy failed, aborting" + exit 1 + } + + create_manifest || { + echo "❌ Manifest creation failed, aborting" + exit 1 + } + + create_checksums || { + echo "❌ Checksum creation failed, aborting" + exit 1 + } + + create_zip_package || { + echo "❌ Zip package creation failed, aborting" + exit 1 + } + + echo "" + echo "========================================" + echo "✅ Offline package build completed" + echo "========================================" + echo "Package contents available at: $OUTPUT_DIR" + if [[ "$COMPRESS" == "true" ]]; then + echo "Compressed package available at: $(cd "$(dirname "$OUTPUT_DIR")" && pwd)/$(offline_package_name).zip" + fi + echo "" +} + +main "$@" diff --git a/deploy/sql/init.sql b/deploy/sql/init.sql new file mode 100644 index 000000000..4dba737bf --- /dev/null +++ b/deploy/sql/init.sql @@ -0,0 +1,445 @@ +-- 1. Create custom Schema (if not exists) +CREATE SCHEMA IF NOT EXISTS nexent; + +-- 2. Switch to the Schema (subsequent operations default to this Schema) +SET search_path TO nexent; + +CREATE TABLE IF NOT EXISTS "conversation_message_t" ( + "message_id" SERIAL, + "conversation_id" int4, + "message_index" int4, + "message_role" varchar(30) COLLATE "pg_catalog"."default", + "message_content" varchar COLLATE "pg_catalog"."default", + "minio_files" varchar, + "opinion_flag" varchar(1), + "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, + "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "created_by" varchar(100) COLLATE "pg_catalog"."default", + "updated_by" varchar(100) COLLATE "pg_catalog"."default", + CONSTRAINT "conversation_message_t_pk" PRIMARY KEY ("message_id") +); +ALTER TABLE "conversation_message_t" OWNER TO "root"; +COMMENT ON COLUMN "conversation_message_t"."conversation_id" IS 'Formal foreign key, used to associate with the conversation'; +COMMENT ON COLUMN "conversation_message_t"."message_index" IS 'Sequence number, used for frontend display sorting'; +COMMENT ON COLUMN "conversation_message_t"."message_role" IS 'Role sending the message, such as system, assistant, user'; +COMMENT ON COLUMN "conversation_message_t"."message_content" IS 'Complete content of the message'; +COMMENT ON COLUMN "conversation_message_t"."minio_files" IS 'Images or documents uploaded by users in the chat interface, stored as a list'; +COMMENT ON COLUMN "conversation_message_t"."opinion_flag" IS 'User feedback on the conversation, enum value Y represents positive, N represents negative'; +COMMENT ON COLUMN "conversation_message_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; +COMMENT ON COLUMN "conversation_message_t"."create_time" IS 'Creation time, audit field'; +COMMENT ON COLUMN "conversation_message_t"."update_time" IS 'Update time, audit field'; +COMMENT ON COLUMN "conversation_message_t"."created_by" IS 'Creator ID, audit field'; +COMMENT ON COLUMN "conversation_message_t"."updated_by" IS 'Last updater ID, audit field'; +COMMENT ON TABLE "conversation_message_t" IS 'Carries specific response message content in conversations'; + +CREATE TABLE IF NOT EXISTS "conversation_message_unit_t" ( + "unit_id" SERIAL, + "message_id" int4, + "conversation_id" int4, + "unit_index" int4, + "unit_type" varchar(100) COLLATE "pg_catalog"."default", + "unit_content" varchar COLLATE "pg_catalog"."default", + "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, + "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "updated_by" varchar(100) COLLATE "pg_catalog"."default", + "created_by" varchar(100) COLLATE "pg_catalog"."default", + CONSTRAINT "conversation_message_unit_t_pk" PRIMARY KEY ("unit_id") +); +ALTER TABLE "conversation_message_unit_t" OWNER TO "root"; +COMMENT ON COLUMN "conversation_message_unit_t"."message_id" IS 'Formal foreign key, used to associate with the message'; +COMMENT ON COLUMN "conversation_message_unit_t"."conversation_id" IS 'Formal foreign key, used to associate with the conversation'; +COMMENT ON COLUMN "conversation_message_unit_t"."unit_index" IS 'Sequence number, used for frontend display sorting'; +COMMENT ON COLUMN "conversation_message_unit_t"."unit_type" IS 'Type of minimum response unit'; +COMMENT ON COLUMN "conversation_message_unit_t"."unit_content" IS 'Complete content of the minimum response unit'; +COMMENT ON COLUMN "conversation_message_unit_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; +COMMENT ON COLUMN "conversation_message_unit_t"."create_time" IS 'Creation time, audit field'; +COMMENT ON COLUMN "conversation_message_unit_t"."update_time" IS 'Update time, audit field'; +COMMENT ON COLUMN "conversation_message_unit_t"."updated_by" IS 'Last updater ID, audit field'; +COMMENT ON COLUMN "conversation_message_unit_t"."created_by" IS 'Creator ID, audit field'; +COMMENT ON TABLE "conversation_message_unit_t" IS 'Carries agent output content in each message'; + +CREATE TABLE IF NOT EXISTS "conversation_record_t" ( + "conversation_id" SERIAL, + "conversation_title" varchar(100) COLLATE "pg_catalog"."default", + "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, + "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "updated_by" varchar(100) COLLATE "pg_catalog"."default", + "created_by" varchar(100) COLLATE "pg_catalog"."default", + CONSTRAINT "conversation_record_t_pk" PRIMARY KEY ("conversation_id") +); +ALTER TABLE "conversation_record_t" OWNER TO "root"; +COMMENT ON COLUMN "conversation_record_t"."conversation_title" IS 'Conversation title'; +COMMENT ON COLUMN "conversation_record_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; +COMMENT ON COLUMN "conversation_record_t"."update_time" IS 'Update time, audit field'; +COMMENT ON COLUMN "conversation_record_t"."create_time" IS 'Creation time, audit field'; +COMMENT ON COLUMN "conversation_record_t"."updated_by" IS 'Last updater ID, audit field'; +COMMENT ON COLUMN "conversation_record_t"."created_by" IS 'Creator ID, audit field'; +COMMENT ON TABLE "conversation_record_t" IS 'Overall information of Q&A conversations'; + +CREATE TABLE IF NOT EXISTS "conversation_source_image_t" ( + "image_id" SERIAL, + "conversation_id" int4, + "message_id" int4, + "unit_id" int4, + "image_url" varchar COLLATE "pg_catalog"."default", + "cite_index" int4, + "search_type" varchar(100) COLLATE "pg_catalog"."default", + "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, + "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "created_by" varchar(100) COLLATE "pg_catalog"."default", + "updated_by" varchar(100) COLLATE "pg_catalog"."default", + CONSTRAINT "conversation_source_image_t_pk" PRIMARY KEY ("image_id") +); +ALTER TABLE "conversation_source_image_t" OWNER TO "root"; +COMMENT ON COLUMN "conversation_source_image_t"."conversation_id" IS 'Formal foreign key, used to associate with the conversation of the search source'; +COMMENT ON COLUMN "conversation_source_image_t"."message_id" IS 'Formal foreign key, used to associate with the conversation message of the search source'; +COMMENT ON COLUMN "conversation_source_image_t"."unit_id" IS 'Formal foreign key, used to associate with the minimum message unit of the search source (if any)'; +COMMENT ON COLUMN "conversation_source_image_t"."image_url" IS 'URL address of the image'; +COMMENT ON COLUMN "conversation_source_image_t"."cite_index" IS '[Reserved] Citation sequence number, used for precise tracing'; +COMMENT ON COLUMN "conversation_source_image_t"."search_type" IS '[Reserved] Search source type, used to distinguish the search tool used for this record, optional values web/local'; +COMMENT ON COLUMN "conversation_source_image_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; +COMMENT ON COLUMN "conversation_source_image_t"."create_time" IS 'Creation time, audit field'; +COMMENT ON COLUMN "conversation_source_image_t"."update_time" IS 'Update time, audit field'; +COMMENT ON COLUMN "conversation_source_image_t"."created_by" IS 'Creator ID, audit field'; +COMMENT ON COLUMN "conversation_source_image_t"."updated_by" IS 'Last updater ID, audit field'; +COMMENT ON TABLE "conversation_source_image_t" IS 'Carries search image source information for conversation messages'; + +CREATE TABLE IF NOT EXISTS "conversation_source_search_t" ( + "search_id" SERIAL, + "unit_id" int4, + "message_id" int4, + "conversation_id" int4, + "source_type" varchar(100) COLLATE "pg_catalog"."default", + "source_title" varchar(400) COLLATE "pg_catalog"."default", + "source_location" varchar(400) COLLATE "pg_catalog"."default", + "source_content" varchar COLLATE "pg_catalog"."default", + "score_overall" numeric(7,6), + "score_accuracy" numeric(7,6), + "score_semantic" numeric(7,6), + "published_date" timestamp(0), + "cite_index" int4, + "search_type" varchar(100) COLLATE "pg_catalog"."default", + "tool_sign" varchar(30) COLLATE "pg_catalog"."default", + "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, + "updated_by" varchar(100) COLLATE "pg_catalog"."default", + "created_by" varchar(100) COLLATE "pg_catalog"."default", + CONSTRAINT "conversation_source_search_t_pk" PRIMARY KEY ("search_id") +); +ALTER TABLE "conversation_source_search_t" OWNER TO "root"; +COMMENT ON COLUMN "conversation_source_search_t"."unit_id" IS 'Formal foreign key, used to associate with the minimum message unit of the search source (if any)'; +COMMENT ON COLUMN "conversation_source_search_t"."message_id" IS 'Formal foreign key, used to associate with the conversation message of the search source'; +COMMENT ON COLUMN "conversation_source_search_t"."conversation_id" IS 'Formal foreign key, used to associate with the conversation of the search source'; +COMMENT ON COLUMN "conversation_source_search_t"."source_type" IS 'Source type, used to distinguish if source_location is URL or path, optional values url/text'; +COMMENT ON COLUMN "conversation_source_search_t"."source_title" IS 'Title or filename of the search source'; +COMMENT ON COLUMN "conversation_source_search_t"."source_location" IS 'URL link or file path of the search source'; +COMMENT ON COLUMN "conversation_source_search_t"."source_content" IS 'Original text of the search source'; +COMMENT ON COLUMN "conversation_source_search_t"."score_overall" IS 'Overall similarity score between source and user query, calculated as weighted average of details'; +COMMENT ON COLUMN "conversation_source_search_t"."score_accuracy" IS 'Accuracy score'; +COMMENT ON COLUMN "conversation_source_search_t"."score_semantic" IS 'Semantic similarity score'; +COMMENT ON COLUMN "conversation_source_search_t"."published_date" IS 'Upload date of local file or network search date'; +COMMENT ON COLUMN "conversation_source_search_t"."cite_index" IS 'Citation sequence number, used for precise tracing'; +COMMENT ON COLUMN "conversation_source_search_t"."search_type" IS 'Search source type, specifically describes the search tool used for this record, optional values web_search/knowledge_base_search'; +COMMENT ON COLUMN "conversation_source_search_t"."tool_sign" IS 'Simple tool identifier, used to distinguish index sources in large model output summary text'; +COMMENT ON COLUMN "conversation_source_search_t"."create_time" IS 'Creation time, audit field'; +COMMENT ON COLUMN "conversation_source_search_t"."update_time" IS 'Update time, audit field'; +COMMENT ON COLUMN "conversation_source_search_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; +COMMENT ON COLUMN "conversation_source_search_t"."updated_by" IS 'Last updater ID, audit field'; +COMMENT ON COLUMN "conversation_source_search_t"."created_by" IS 'Creator ID, audit field'; +COMMENT ON TABLE "conversation_source_search_t" IS 'Carries search text source information referenced in conversation response messages'; + +CREATE TABLE IF NOT EXISTS "model_record_t" ( + "model_id" SERIAL, + "model_repo" varchar(100) COLLATE "pg_catalog"."default", + "model_name" varchar(100) COLLATE "pg_catalog"."default" NOT NULL, + "model_factory" varchar(100) COLLATE "pg_catalog"."default", + "model_type" varchar(100) COLLATE "pg_catalog"."default", + "api_key" varchar(500) COLLATE "pg_catalog"."default", + "base_url" varchar(500) COLLATE "pg_catalog"."default", + "max_tokens" int4, + "used_token" int4, + "display_name" varchar(100) COLLATE "pg_catalog"."default", + "connect_status" varchar(100) COLLATE "pg_catalog"."default", + "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, + "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "updated_by" varchar(100) COLLATE "pg_catalog"."default", + "created_by" varchar(100) COLLATE "pg_catalog"."default", + CONSTRAINT "nexent_models_t_pk" PRIMARY KEY ("model_id") +); +ALTER TABLE "model_record_t" OWNER TO "root"; +COMMENT ON COLUMN "model_record_t"."model_id" IS 'Model ID, unique primary key'; +COMMENT ON COLUMN "model_record_t"."model_repo" IS 'Model path address'; +COMMENT ON COLUMN "model_record_t"."model_name" IS 'Model name'; +COMMENT ON COLUMN "model_record_t"."model_factory" IS 'Model manufacturer, determines specific format of api-key and model response. Currently defaults to OpenAI-API-Compatible'; +COMMENT ON COLUMN "model_record_t"."model_type" IS 'Model type, e.g. chat, embedding, rerank, tts, asr'; +COMMENT ON COLUMN "model_record_t"."api_key" IS 'Model API key, used for authentication for some models'; +COMMENT ON COLUMN "model_record_t"."base_url" IS 'Base URL address, used for requesting remote model services'; +COMMENT ON COLUMN "model_record_t"."max_tokens" IS 'Maximum available tokens for the model'; +COMMENT ON COLUMN "model_record_t"."used_token" IS 'Number of tokens already used by the model in Q&A'; +COMMENT ON COLUMN "model_record_t"."display_name" IS 'Model name displayed directly in frontend, customized by user'; +COMMENT ON COLUMN "model_record_t"."connect_status" IS 'Model connectivity status from last check, optional values: "检测中"、"可用"、"不可用"'; +COMMENT ON COLUMN "model_record_t"."create_time" IS 'Creation time, audit field'; +COMMENT ON COLUMN "model_record_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; +COMMENT ON COLUMN "model_record_t"."update_time" IS 'Update time, audit field'; +COMMENT ON COLUMN "model_record_t"."updated_by" IS 'Last updater ID, audit field'; +COMMENT ON COLUMN "model_record_t"."created_by" IS 'Creator ID, audit field'; +COMMENT ON TABLE "model_record_t" IS 'List of models defined by users in the configuration page'; + +INSERT INTO "nexent"."model_record_t" ("model_repo", "model_name", "model_factory", "model_type", "api_key", "base_url", "max_tokens", "used_token", "display_name", "connect_status") +SELECT '', 'volcano_tts', 'OpenAI-API-Compatible', 'tts', '', '', 0, 0, 'volcano_tts', 'unavailable' +WHERE NOT EXISTS ( + SELECT 1 FROM "nexent"."model_record_t" + WHERE "model_name" = 'volcano_tts' AND "model_type" = 'tts' +); +INSERT INTO "nexent"."model_record_t" ("model_repo", "model_name", "model_factory", "model_type", "api_key", "base_url", "max_tokens", "used_token", "display_name", "connect_status") +SELECT '', 'volcano_stt', 'OpenAI-API-Compatible', 'stt', '', '', 0, 0, 'volcano_stt', 'unavailable' +WHERE NOT EXISTS ( + SELECT 1 FROM "nexent"."model_record_t" + WHERE "model_name" = 'volcano_stt' AND "model_type" = 'stt' +); + +CREATE TABLE IF NOT EXISTS "knowledge_record_t" ( + "knowledge_id" SERIAL, + "index_name" varchar(100) COLLATE "pg_catalog"."default", + "knowledge_describe" varchar(300) COLLATE "pg_catalog"."default", + "tenant_id" varchar(100) COLLATE "pg_catalog"."default", + "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, + "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, + "updated_by" varchar(100) COLLATE "pg_catalog"."default", + "created_by" varchar(100) COLLATE "pg_catalog"."default", + CONSTRAINT "knowledge_record_t_pk" PRIMARY KEY ("knowledge_id") +); +ALTER TABLE "knowledge_record_t" OWNER TO "root"; +COMMENT ON COLUMN "knowledge_record_t"."knowledge_id" IS 'Knowledge base ID, unique primary key'; +COMMENT ON COLUMN "knowledge_record_t"."index_name" IS 'Knowledge base name'; +COMMENT ON COLUMN "knowledge_record_t"."knowledge_describe" IS 'Knowledge base description'; +COMMENT ON COLUMN "knowledge_record_t"."tenant_id" IS 'Tenant ID'; +COMMENT ON COLUMN "knowledge_record_t"."create_time" IS 'Creation time, audit field'; +COMMENT ON COLUMN "knowledge_record_t"."update_time" IS 'Update time, audit field'; +COMMENT ON COLUMN "knowledge_record_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; +COMMENT ON COLUMN "knowledge_record_t"."updated_by" IS 'Last updater ID, audit field'; +COMMENT ON COLUMN "knowledge_record_t"."created_by" IS 'Creator ID, audit field'; +COMMENT ON TABLE "knowledge_record_t" IS 'Records knowledge base description and status information'; + +-- Create the ag_tool_info_t table +CREATE TABLE IF NOT EXISTS nexent.ag_tool_info_t ( + tool_id SERIAL PRIMARY KEY NOT NULL, + name VARCHAR(100), + class_name VARCHAR(100), + description VARCHAR, + source VARCHAR(100), + author VARCHAR(100), + usage VARCHAR(100), + params JSON, + inputs VARCHAR, + output_type VARCHAR(100), + is_available BOOLEAN DEFAULT FALSE, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Trigger to update update_time when the record is modified +CREATE OR REPLACE FUNCTION update_ag_tool_info_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS update_ag_tool_info_update_time_trigger ON nexent.ag_tool_info_t; +CREATE TRIGGER update_ag_tool_info_update_time_trigger +BEFORE UPDATE ON nexent.ag_tool_info_t +FOR EACH ROW +EXECUTE FUNCTION update_ag_tool_info_update_time(); + +-- Add comment to the table +COMMENT ON TABLE nexent.ag_tool_info_t IS 'Information table for prompt tools'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.ag_tool_info_t.tool_id IS 'ID'; +COMMENT ON COLUMN nexent.ag_tool_info_t.name IS 'Unique key name'; +COMMENT ON COLUMN nexent.ag_tool_info_t.class_name IS 'Tool class name, used when the tool is instantiated'; +COMMENT ON COLUMN nexent.ag_tool_info_t.description IS 'Prompt tool description'; +COMMENT ON COLUMN nexent.ag_tool_info_t.source IS 'Source'; +COMMENT ON COLUMN nexent.ag_tool_info_t.author IS 'Tool author'; +COMMENT ON COLUMN nexent.ag_tool_info_t.usage IS 'Usage'; +COMMENT ON COLUMN nexent.ag_tool_info_t.params IS 'Tool parameter information (json)'; +COMMENT ON COLUMN nexent.ag_tool_info_t.inputs IS 'Prompt tool inputs description'; +COMMENT ON COLUMN nexent.ag_tool_info_t.output_type IS 'Prompt tool output description'; +COMMENT ON COLUMN nexent.ag_tool_info_t.is_available IS 'Whether the tool can be used under the current main service'; +COMMENT ON COLUMN nexent.ag_tool_info_t.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.ag_tool_info_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.ag_tool_info_t.created_by IS 'Creator'; +COMMENT ON COLUMN nexent.ag_tool_info_t.updated_by IS 'Updater'; +COMMENT ON COLUMN nexent.ag_tool_info_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; + +-- Create the ag_tenant_agent_t table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.ag_tenant_agent_t ( + agent_id SERIAL PRIMARY KEY NOT NULL, + name VARCHAR(100), + description VARCHAR, + business_description VARCHAR, + model_name VARCHAR(100), + max_steps INTEGER, + prompt TEXT, + parent_agent_id INTEGER, + tenant_id VARCHAR(100), + enabled BOOLEAN DEFAULT FALSE, + provide_run_summary BOOLEAN DEFAULT FALSE, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Create a function to update the update_time column +CREATE OR REPLACE FUNCTION update_ag_tenant_agent_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Create a trigger to call the function before each update +DROP TRIGGER IF EXISTS update_ag_tenant_agent_update_time_trigger ON nexent.ag_tenant_agent_t; +CREATE TRIGGER update_ag_tenant_agent_update_time_trigger +BEFORE UPDATE ON nexent.ag_tenant_agent_t +FOR EACH ROW +EXECUTE FUNCTION update_ag_tenant_agent_update_time(); +-- Add comments to the table +COMMENT ON TABLE nexent.ag_tenant_agent_t IS 'Information table for agents'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.ag_tenant_agent_t.agent_id IS 'ID'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.name IS 'Agent name'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.description IS 'Description'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.business_description IS 'Manually entered by the user to describe the entire business process'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.model_name IS 'Name of the model used'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.max_steps IS 'Maximum number of steps'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.parent_agent_id IS 'Parent Agent ID'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.tenant_id IS 'Belonging tenant'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.enabled IS 'Enable flag'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.provide_run_summary IS 'Whether to provide the running summary to the manager agent'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.created_by IS 'Creator'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.updated_by IS 'Updater'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; + +-- Create the ag_user_agent_t table in the nexent schema with new fields +CREATE TABLE IF NOT EXISTS nexent.ag_user_agent_t ( + user_agent_id SERIAL PRIMARY KEY NOT NULL, + agent_id INTEGER, + prompt TEXT, + tenant_id VARCHAR(100), + user_id VARCHAR(100), + enabled BOOLEAN DEFAULT FALSE, + provide_run_summary BOOLEAN DEFAULT FALSE, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Add comment to the table +COMMENT ON TABLE nexent.ag_user_agent_t IS 'Information table for user agents'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.ag_user_agent_t.user_agent_id IS 'ID'; +COMMENT ON COLUMN nexent.ag_user_agent_t.agent_id IS 'Agent ID'; +COMMENT ON COLUMN nexent.ag_user_agent_t.prompt IS 'System prompt'; +COMMENT ON COLUMN nexent.ag_user_agent_t.tenant_id IS 'Belonging tenant'; +COMMENT ON COLUMN nexent.ag_user_agent_t.user_id IS 'User ID'; +COMMENT ON COLUMN nexent.ag_user_agent_t.enabled IS 'Enable flag'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.provide_run_summary IS 'Whether to provide the running summary to the manager agent'; +COMMENT ON COLUMN nexent.ag_user_agent_t.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.ag_user_agent_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.ag_user_agent_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; + +-- Create a function to update the update_time column +CREATE OR REPLACE FUNCTION update_ag_user_agent_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Add comment to the function +COMMENT ON FUNCTION update_ag_user_agent_update_time() IS 'Function to update the update_time column when a record in ag_user_agent_t is updated'; + +-- Create a trigger to call the function before each update +DROP TRIGGER IF EXISTS update_ag_user_agent_update_time_trigger ON nexent.ag_user_agent_t; +CREATE TRIGGER update_ag_user_agent_update_time_trigger +BEFORE UPDATE ON nexent.ag_user_agent_t +FOR EACH ROW +EXECUTE FUNCTION update_ag_user_agent_update_time(); + +-- Add comment to the trigger +COMMENT ON TRIGGER update_ag_user_agent_update_time_trigger ON nexent.ag_user_agent_t IS 'Trigger to call update_ag_user_agent_update_time function before each update on ag_user_agent_t table'; + +-- Create the ag_tool_instance_t table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.ag_tool_instance_t ( + tool_instance_id SERIAL PRIMARY KEY NOT NULL, + tool_id INTEGER, + agent_id INTEGER, + params JSON, + user_id VARCHAR(100), + tenant_id VARCHAR(100), + enabled BOOLEAN DEFAULT FALSE, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Add comment to the table +COMMENT ON TABLE nexent.ag_tool_instance_t IS 'Information table for tenant tool configuration.'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.ag_tool_instance_t.tool_instance_id IS 'ID'; +COMMENT ON COLUMN nexent.ag_tool_instance_t.tool_id IS 'Tenant tool ID'; +COMMENT ON COLUMN nexent.ag_tool_instance_t.agent_id IS 'Agent ID'; +COMMENT ON COLUMN nexent.ag_tool_instance_t.params IS 'Parameter configuration'; +COMMENT ON COLUMN nexent.ag_tool_instance_t.user_id IS 'User ID'; +COMMENT ON COLUMN nexent.ag_tool_instance_t.tenant_id IS 'Tenant ID'; +COMMENT ON COLUMN nexent.ag_tool_instance_t.enabled IS 'Enable flag'; +COMMENT ON COLUMN nexent.ag_tool_instance_t.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.ag_tool_instance_t.update_time IS 'Update time'; + +-- Create a function to update the update_time column +CREATE OR REPLACE FUNCTION update_ag_tool_instance_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Add comment to the function +COMMENT ON FUNCTION update_ag_tool_instance_update_time() IS 'Function to update the update_time column when a record in ag_tool_instance_t is updated'; + +-- Create a trigger to call the function before each update +DROP TRIGGER IF EXISTS update_ag_tool_instance_update_time_trigger ON nexent.ag_tool_instance_t; +CREATE TRIGGER update_ag_tool_instance_update_time_trigger +BEFORE UPDATE ON nexent.ag_tool_instance_t +FOR EACH ROW +EXECUTE FUNCTION update_ag_tool_instance_update_time(); + +-- Add comment to the trigger +COMMENT ON TRIGGER update_ag_tool_instance_update_time_trigger ON nexent.ag_tool_instance_t IS 'Trigger to call update_ag_tool_instance_update_time function before each update on ag_tool_instance_t table'; diff --git a/deploy/sql/migrations/README.md b/deploy/sql/migrations/README.md new file mode 100644 index 000000000..5c18bf2c0 --- /dev/null +++ b/deploy/sql/migrations/README.md @@ -0,0 +1,19 @@ +# SQL Migration Layout + +Nexent keeps deployment SQL in versioned migration files under this directory. +The migration runner uses the SQL file name as the migration ID and stores the +current file checksum in `nexent.schema_migrations`. + +Execution rules: + +- Files are discovered with `*.sql` and sorted by version-aware filename order. +- A file with no migration record is executed and recorded as `applied`. +- A file with the same recorded checksum is skipped. +- A file with a different recorded checksum is executed again, then its checksum, + execution time, app version, and source file are updated. + +Keep migration SQL idempotent because changing an existing file causes it to run +again. Use patterns such as `CREATE TABLE IF NOT EXISTS`, `ALTER TABLE ... ADD +COLUMN IF NOT EXISTS`, and conflict-safe inserts where possible. + +`deploy/sql/init.sql` is the initial baseline before these incremental files. diff --git a/deploy/sql/migrations/v1_merged_migrations.sql b/deploy/sql/migrations/v1_merged_migrations.sql new file mode 100644 index 000000000..b56200d3c --- /dev/null +++ b/deploy/sql/migrations/v1_merged_migrations.sql @@ -0,0 +1,1354 @@ +-- Nexent merged SQL migrations: v1 +-- This file is generated from historical migration files. + +-- 1. 为knowledge_record_t表添加knowledge_sources�? +ALTER TABLE nexent.knowledge_record_t +ADD COLUMN IF NOT EXISTS "knowledge_sources" varchar(100) COLLATE "pg_catalog"."default"; + +-- 添加列注释 +COMMENT ON COLUMN nexent.knowledge_record_t."knowledge_sources" IS 'Knowledge base sources'; + + +-- 2. 创建tenant_config_t表 +CREATE TABLE IF NOT EXISTS nexent.tenant_config_t ( + tenant_config_id SERIAL PRIMARY KEY NOT NULL, + tenant_id VARCHAR(100), + user_id VARCHAR(100), + value_type VARCHAR(100), + config_key VARCHAR(100), + config_value TEXT, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- 添加表注释 +COMMENT ON TABLE nexent.tenant_config_t IS 'Tenant configuration information table'; + +-- 添加列注释 +COMMENT ON COLUMN nexent.tenant_config_t.tenant_config_id IS 'ID'; +COMMENT ON COLUMN nexent.tenant_config_t.tenant_id IS 'Tenant ID'; +COMMENT ON COLUMN nexent.tenant_config_t.user_id IS 'User ID'; +COMMENT ON COLUMN nexent.tenant_config_t.value_type IS 'Value type'; +COMMENT ON COLUMN nexent.tenant_config_t.config_key IS 'Config key'; +COMMENT ON COLUMN nexent.tenant_config_t.config_value IS 'Config value'; +COMMENT ON COLUMN nexent.tenant_config_t.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.tenant_config_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.tenant_config_t.created_by IS 'Creator'; +COMMENT ON COLUMN nexent.tenant_config_t.updated_by IS 'Updater'; +COMMENT ON COLUMN nexent.tenant_config_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; + +-- 创建更新update_time的函�? +CREATE OR REPLACE FUNCTION update_tenant_config_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- 添加函数注释 +COMMENT ON FUNCTION update_tenant_config_update_time() IS 'Function to update the update_time column when a record in tenant_config_t is updated'; + +-- 创建触发器 +DROP TRIGGER IF EXISTS update_tenant_config_update_time_trigger ON nexent.tenant_config_t; +CREATE TRIGGER update_tenant_config_update_time_trigger +BEFORE UPDATE ON nexent.tenant_config_t +FOR EACH ROW +EXECUTE FUNCTION update_tenant_config_update_time(); + +-- 添加触发器注释 +COMMENT ON TRIGGER update_tenant_config_update_time_trigger ON nexent.tenant_config_t +IS 'Trigger to call update_tenant_config_update_time function before each update on tenant_config_t table'; + +ALTER TABLE model_record_t +ADD COLUMN IF NOT EXISTS tenant_id varchar(100) COLLATE pg_catalog.default DEFAULT 'tenant_id'; +COMMENT ON COLUMN "model_record_t"."tenant_id" IS 'Tenant ID for filtering'; + +-- Incremental SQL to alter config_value column type in nexent.tenant_config_t table + +-- Check if the table exists before attempting to alter it +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'nexent' + AND table_name = 'tenant_config_t' + ) THEN + -- Use TEXT so existing large config values are preserved + EXECUTE 'ALTER TABLE nexent.tenant_config_t ALTER COLUMN config_value TYPE TEXT'; + + -- Log the change + RAISE NOTICE 'Altered config_value column type to TEXT in nexent.tenant_config_t'; + ELSE + RAISE NOTICE 'Table nexent.tenant_config_t does not exist, skipping alteration'; + END IF; +END $$; + +-- Migration: Add mcp_record_t table +-- Date: 2024-06-30 +-- Description: Create MCP (Model Context Protocol) records table with audit fields + +-- Set search path to nexent schema +SET search_path TO nexent; + +-- Create the mcp_record_t table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.mcp_record_t ( + mcp_id SERIAL PRIMARY KEY NOT NULL, + tenant_id VARCHAR(100), + user_id VARCHAR(100), + mcp_name VARCHAR(100), + mcp_server VARCHAR(500), + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE "mcp_record_t" OWNER TO "root"; + +-- Add comment to the table +COMMENT ON TABLE nexent.mcp_record_t IS 'MCP (Model Context Protocol) records table'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.mcp_record_t.mcp_id IS 'MCP record ID, unique primary key'; +COMMENT ON COLUMN nexent.mcp_record_t.tenant_id IS 'Tenant ID'; +COMMENT ON COLUMN nexent.mcp_record_t.user_id IS 'User ID'; +COMMENT ON COLUMN nexent.mcp_record_t.mcp_name IS 'MCP name'; +COMMENT ON COLUMN nexent.mcp_record_t.mcp_server IS 'MCP server address'; +COMMENT ON COLUMN nexent.mcp_record_t.create_time IS 'Creation time, audit field'; +COMMENT ON COLUMN nexent.mcp_record_t.update_time IS 'Update time, audit field'; +COMMENT ON COLUMN nexent.mcp_record_t.created_by IS 'Creator ID, audit field'; +COMMENT ON COLUMN nexent.mcp_record_t.updated_by IS 'Last updater ID, audit field'; +COMMENT ON COLUMN nexent.mcp_record_t.delete_flag IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; + +-- Create a function to update the update_time column +CREATE OR REPLACE FUNCTION update_mcp_record_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Add comment to the function +COMMENT ON FUNCTION update_mcp_record_update_time() IS 'Function to update the update_time column when a record in mcp_record_t is updated'; + +-- Create a trigger to call the function before each update +DROP TRIGGER IF EXISTS update_mcp_record_update_time_trigger ON nexent.mcp_record_t; +CREATE TRIGGER update_mcp_record_update_time_trigger +BEFORE UPDATE ON nexent.mcp_record_t +FOR EACH ROW +EXECUTE FUNCTION update_mcp_record_update_time(); + +-- Add comment to the trigger +COMMENT ON TRIGGER update_mcp_record_update_time_trigger ON nexent.mcp_record_t IS 'Trigger to call update_mcp_record_update_time function before each update on mcp_record_t table'; + +-- Create user tenant relationship table +CREATE TABLE IF NOT EXISTS nexent.user_tenant_t ( + user_tenant_id SERIAL PRIMARY KEY, + user_id VARCHAR(100) NOT NULL, + tenant_id VARCHAR(100) NOT NULL, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag CHAR(1) DEFAULT 'N', + UNIQUE(user_id, tenant_id) +); + +-- Add comment +COMMENT ON TABLE nexent.user_tenant_t IS 'User tenant relationship table'; +COMMENT ON COLUMN nexent.user_tenant_t.user_tenant_id IS 'User tenant relationship ID, primary key'; +COMMENT ON COLUMN nexent.user_tenant_t.user_id IS 'User ID'; +COMMENT ON COLUMN nexent.user_tenant_t.tenant_id IS 'Tenant ID'; +COMMENT ON COLUMN nexent.user_tenant_t.create_time IS 'Create time'; +COMMENT ON COLUMN nexent.user_tenant_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.user_tenant_t.created_by IS 'Created by'; +COMMENT ON COLUMN nexent.user_tenant_t.updated_by IS 'Updated by'; +COMMENT ON COLUMN nexent.user_tenant_t.delete_flag IS 'Delete flag, Y/N'; + +ALTER TABLE nexent.knowledge_record_t + ALTER COLUMN knowledge_describe TYPE varchar(3000); + +ALTER TABLE nexent.mcp_record_t +ADD COLUMN IF NOT EXISTS status BOOLEAN DEFAULT NULL; +COMMENT ON COLUMN nexent.mcp_record_t.status IS 'MCP server connection status, true=connected, false=disconnected, null=unknown'; + +-- Migration script to add new prompt fields to ag_tenant_agent_t table +-- Add three new columns for storing segmented prompt content + +-- Add duty_prompt column +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS duty_prompt TEXT; + +-- Add constraint_prompt column +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS constraint_prompt TEXT; + +-- Add few_shots_prompt column +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS few_shots_prompt TEXT; + +-- Drop prompt column +ALTER TABLE nexent.ag_tenant_agent_t +DROP COLUMN IF EXISTS prompt; + +-- Add comments to the new columns +COMMENT ON COLUMN nexent.ag_tenant_agent_t.duty_prompt IS 'Duty prompt content'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.constraint_prompt IS 'Constraint prompt content'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.few_shots_prompt IS 'Few shots prompt content'; + +-- Migration script to add ag_agent_relation_t table for recording agent parent-child relationships +-- This table is used to store the hierarchical relationships between agents + +-- Create the ag_agent_relation_t table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.ag_agent_relation_t ( + relation_id SERIAL PRIMARY KEY NOT NULL, + selected_agent_id INTEGER, + parent_agent_id INTEGER, + tenant_id VARCHAR(100), + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Create a function to update the update_time column +CREATE OR REPLACE FUNCTION update_ag_agent_relation_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Create a trigger to call the function before each update +DROP TRIGGER IF EXISTS update_ag_agent_relation_update_time_trigger ON nexent.ag_agent_relation_t; +CREATE TRIGGER update_ag_agent_relation_update_time_trigger +BEFORE UPDATE ON nexent.ag_agent_relation_t +FOR EACH ROW +EXECUTE FUNCTION update_ag_agent_relation_update_time(); + +-- Add comment to the table +COMMENT ON TABLE nexent.ag_agent_relation_t IS 'Agent parent-child relationship table'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.ag_agent_relation_t.relation_id IS 'Relationship ID, primary key'; +COMMENT ON COLUMN nexent.ag_agent_relation_t.selected_agent_id IS 'Selected agent ID'; +COMMENT ON COLUMN nexent.ag_agent_relation_t.parent_agent_id IS 'Parent agent ID'; +COMMENT ON COLUMN nexent.ag_agent_relation_t.tenant_id IS 'Tenant ID'; +COMMENT ON COLUMN nexent.ag_agent_relation_t.create_time IS 'Creation time, audit field'; +COMMENT ON COLUMN nexent.ag_agent_relation_t.update_time IS 'Update time, audit field'; +COMMENT ON COLUMN nexent.ag_agent_relation_t.created_by IS 'Creator ID, audit field'; +COMMENT ON COLUMN nexent.ag_agent_relation_t.updated_by IS 'Last updater ID, audit field'; +COMMENT ON COLUMN nexent.ag_agent_relation_t.delete_flag IS 'Delete flag, set to Y for soft delete, optional values Y/N'; + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS is_deep_thinking BOOLEAN DEFAULT FALSE; +COMMENT ON COLUMN nexent.model_record_t.is_deep_thinking IS 'deep thinking switch, true=open, false=close'; + +-- 创建序列 +CREATE SEQUENCE IF NOT EXISTS "nexent"."memory_user_config_t_config_id_seq" +INCREMENT 1 +MINVALUE 1 +MAXVALUE 2147483647 +START 1 +CACHE 1; + + +-- 创建表 +CREATE TABLE IF NOT EXISTS "nexent"."memory_user_config_t" ( + "config_id" SERIAL PRIMARY KEY NOT NULL, + "tenant_id" varchar(100) COLLATE "pg_catalog"."default", + "user_id" varchar(100) COLLATE "pg_catalog"."default", + "value_type" varchar(100) COLLATE "pg_catalog"."default", + "config_key" varchar(100) COLLATE "pg_catalog"."default", + "config_value" varchar(100) COLLATE "pg_catalog"."default", + "create_time" timestamp(6) DEFAULT CURRENT_TIMESTAMP, + "update_time" timestamp(6) DEFAULT CURRENT_TIMESTAMP, + "created_by" varchar(100) COLLATE "pg_catalog"."default", + "updated_by" varchar(100) COLLATE "pg_catalog"."default", + "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying +); + +-- 设置表所有者 +ALTER TABLE "nexent"."memory_user_config_t" OWNER TO "root"; + +COMMENT ON COLUMN "nexent"."memory_user_config_t"."config_id" IS 'ID'; +COMMENT ON COLUMN "nexent"."memory_user_config_t"."tenant_id" IS 'Tenant ID'; +COMMENT ON COLUMN "nexent"."memory_user_config_t"."user_id" IS 'User ID'; +COMMENT ON COLUMN "nexent"."memory_user_config_t"."value_type" IS 'Value type. Optional values: single/multi'; +COMMENT ON COLUMN "nexent"."memory_user_config_t"."config_key" IS 'Config key'; +COMMENT ON COLUMN "nexent"."memory_user_config_t"."config_value" IS 'Config value'; +COMMENT ON COLUMN "nexent"."memory_user_config_t"."create_time" IS 'Creation time'; +COMMENT ON COLUMN "nexent"."memory_user_config_t"."update_time" IS 'Update time'; +COMMENT ON COLUMN "nexent"."memory_user_config_t"."created_by" IS 'Creator'; +COMMENT ON COLUMN "nexent"."memory_user_config_t"."updated_by" IS 'Updater'; +COMMENT ON COLUMN "nexent"."memory_user_config_t"."delete_flag" IS 'Whether it is deleted. Optional values: Y/N'; + +COMMENT ON TABLE "nexent"."memory_user_config_t" IS 'User configuration of memory setting table'; + +CREATE OR REPLACE FUNCTION "update_memory_user_config_update_time"() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS "update_memory_user_config_update_time_trigger" ON "nexent"."memory_user_config_t"; +CREATE TRIGGER "update_memory_user_config_update_time_trigger" +BEFORE UPDATE ON "nexent"."memory_user_config_t" +FOR EACH ROW +EXECUTE FUNCTION "update_memory_user_config_update_time"(); + +CREATE SEQUENCE IF NOT EXISTS "nexent"."partner_mapping_id_t_mapping_id_seq" +INCREMENT 1 +MINVALUE 1 +MAXVALUE 2147483647 +START 1 +CACHE 1; + +CREATE TABLE IF NOT EXISTS "nexent"."partner_mapping_id_t" ( + "mapping_id" serial PRIMARY KEY NOT NULL, + "external_id" varchar(100) COLLATE "pg_catalog"."default", + "internal_id" int4, + "mapping_type" varchar(30) COLLATE "pg_catalog"."default", + "tenant_id" varchar(100) COLLATE "pg_catalog"."default", + "user_id" varchar(100) COLLATE "pg_catalog"."default", + "create_time" timestamp(6) DEFAULT CURRENT_TIMESTAMP, + "update_time" timestamp(6) DEFAULT CURRENT_TIMESTAMP, + "created_by" varchar(100) COLLATE "pg_catalog"."default", + "updated_by" varchar(100) COLLATE "pg_catalog"."default", + "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying +); + +ALTER TABLE "nexent"."partner_mapping_id_t" OWNER TO "root"; + +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."mapping_id" IS 'ID'; +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."external_id" IS 'The external id given by the outer partner'; +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."internal_id" IS 'The internal id of the other database table'; +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."mapping_type" IS 'Type of the external - internal mapping, value set: CONVERSATION'; +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."tenant_id" IS 'Tenant ID'; +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."user_id" IS 'User ID'; +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."create_time" IS 'Creation time'; +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."update_time" IS 'Update time'; +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."created_by" IS 'Creator'; +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."updated_by" IS 'Updater'; +COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."delete_flag" IS 'Whether it is deleted. Optional values: Y/N'; + +CREATE OR REPLACE FUNCTION "update_partner_mapping_update_time"() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS "update_partner_mapping_update_time_trigger" ON "nexent"."partner_mapping_id_t"; +CREATE TRIGGER "update_partner_mapping_update_time_trigger" +BEFORE UPDATE ON "nexent"."partner_mapping_id_t" +FOR EACH ROW +EXECUTE FUNCTION "update_partner_mapping_update_time"(); + +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS display_name VARCHAR(100); +COMMENT ON COLUMN nexent.ag_tenant_agent_t.display_name IS 'Agent展示名称'; + +ALTER TABLE nexent.model_record_t +DROP COLUMN IF EXISTS is_deep_thinking; + +-- Add model_name column to knowledge_record_t table, used to record the embedding model used by the knowledge base + +-- Switch to nexent schema +SET search_path TO nexent; + +-- Add model_name column +ALTER TABLE "knowledge_record_t" +ADD COLUMN IF NOT EXISTS "embedding_model_name" varchar(200) COLLATE "pg_catalog"."default"; + +-- Add column comment +COMMENT ON COLUMN "knowledge_record_t"."embedding_model_name" IS 'Embedding model name, used to record the embedding model used by the knowledge base'; + +-- Add origin_name column to ag_tool_info_t table +-- This field stores the original tool name before any transformations + +ALTER TABLE nexent.ag_tool_info_t +ADD COLUMN IF NOT EXISTS origin_name VARCHAR(100); + +-- Add comment to document the purpose of this field +COMMENT ON COLUMN nexent.ag_tool_info_t.origin_name IS 'Original tool name before any transformations or mappings'; + +-- Add category column to ag_tool_info_t table +-- This field stores the tool category information (search, file, email, terminal) + +ALTER TABLE nexent.ag_tool_info_t +ADD COLUMN IF NOT EXISTS category VARCHAR(100); + +-- Add comment to document the purpose of this field +COMMENT ON COLUMN nexent.ag_tool_info_t.category IS 'Tool category information'; + +-- Add model_id column to ag_tenant_agent_t table and deprecate model_name field +-- Date: 2024-09-28 +-- Description: Add model_id field to ag_tenant_agent_t table and mark model_name as deprecated + +-- Switch to the nexent schema +SET search_path TO nexent; + +-- Add model_id column to ag_tenant_agent_t table +ALTER TABLE ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS model_id INTEGER; + +-- Add comment for the new model_id column +COMMENT ON COLUMN ag_tenant_agent_t.model_id IS 'Model ID, foreign key reference to model_record_t.model_id'; + +-- Update comment for model_name column to mark it as deprecated +COMMENT ON COLUMN ag_tenant_agent_t.model_name IS '[DEPRECATED] Name of the model used, use model_id instead'; + +-- Optional: Add foreign key constraint (uncomment if needed) +-- ALTER TABLE ag_tenant_agent_t +-- ADD CONSTRAINT fk_ag_tenant_agent_model_id +-- FOREIGN KEY (model_id) REFERENCES model_record_t(model_id); + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS expected_chunk_size INT4, +ADD COLUMN IF NOT EXISTS maximum_chunk_size INT4; + +COMMENT ON COLUMN nexent.model_record_t.expected_chunk_size IS 'Expected chunk size for embedding models, used during document chunking'; +COMMENT ON COLUMN nexent.model_record_t.maximum_chunk_size IS 'Maximum chunk size for embedding models, used during document chunking'; + + +-- Add business_logic_model_name and business_logic_model_id fields to ag_tenant_agent_t table +-- These fields store the LLM model used for generating business logic prompts + +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS business_logic_model_name VARCHAR(100); + +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS business_logic_model_id INTEGER; + +COMMENT ON COLUMN nexent.ag_tenant_agent_t.business_logic_model_name IS 'Model name used for business logic prompt generation'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.business_logic_model_id IS 'Model ID used for business logic prompt generation, foreign key reference to model_record_t.model_id'; + + +ALTER TABLE nexent.tenant_config_t ALTER COLUMN config_value TYPE TEXT; + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS ssl_verify BOOLEAN DEFAULT TRUE; + +COMMENT ON COLUMN nexent.model_record_t.ssl_verify IS 'Whether to verify SSL certificates when connecting to this model API. Default is true. Set to false for local services without SSL support.'; + + +-- Add knowledge_name column if it does not exist +ALTER TABLE nexent.knowledge_record_t +ADD COLUMN IF NOT EXISTS knowledge_name varchar(100) COLLATE "pg_catalog"."default"; + +COMMENT ON COLUMN nexent.knowledge_record_t.knowledge_name IS 'User-facing knowledge base name (display name), mapped to internal index_name'; +COMMENT ON COLUMN nexent.knowledge_record_t.index_name IS 'Internal Elasticsearch index name'; + +-- Backfill existing records: for legacy data, use index_name as knowledge_name +UPDATE nexent.knowledge_record_t +SET knowledge_name = index_name +WHERE knowledge_name IS NULL; + + +-- Add chunk_batch column in model_record_t table +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS chunk_batch INT4; + +COMMENT ON COLUMN nexent.model_record_t.chunk_batch IS 'Batch size for concurrent embedding requests during document chunking'; + +-- Add author column to ag_tenant_agent_t table +-- This migration adds the author field to support agent author information + +-- Add author column with default NULL value for backward compatibility +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS author VARCHAR(100); + +-- Add comment to the column +COMMENT ON COLUMN nexent.ag_tenant_agent_t.author IS 'Agent author'; + + +-- Add invitation code and group management system +-- This migration adds invitation codes, groups, and permission management features + +-- 1. Create tenant_invitation_code_t table for invitation codes +CREATE TABLE IF NOT EXISTS nexent.tenant_invitation_code_t ( + invitation_id SERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + invitation_code VARCHAR(100) NOT NULL, + group_ids VARCHAR, -- int4 list + capacity INT4 NOT NULL DEFAULT 1, + expiry_date TIMESTAMP(6) WITHOUT TIME ZONE, + status VARCHAR(30) NOT NULL, + code_type VARCHAR(30) NOT NULL, + create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), + update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Add comments for tenant_invitation_code_t table +COMMENT ON TABLE nexent.tenant_invitation_code_t IS 'Tenant invitation code information table'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.invitation_id IS 'Invitation ID, primary key'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.tenant_id IS 'Tenant ID, foreign key'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.invitation_code IS 'Invitation code'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.group_ids IS 'Associated group IDs list'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.capacity IS 'Invitation code capacity'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.expiry_date IS 'Invitation code expiry date'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.status IS 'Invitation code status: IN_USE, EXPIRE, DISABLE, RUN_OUT'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.code_type IS 'Invitation code type: ADMIN_INVITE, DEV_INVITE, USER_INVITE'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.create_time IS 'Create time'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.created_by IS 'Created by'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.updated_by IS 'Updated by'; +COMMENT ON COLUMN nexent.tenant_invitation_code_t.delete_flag IS 'Delete flag, Y/N'; + +-- 2. Create tenant_invitation_record_t table for invitation usage records +CREATE TABLE IF NOT EXISTS nexent.tenant_invitation_record_t ( + invitation_record_id SERIAL PRIMARY KEY, + invitation_id INT4 NOT NULL, + user_id VARCHAR(100) NOT NULL, + create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), + update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Add comments for tenant_invitation_record_t table +COMMENT ON TABLE nexent.tenant_invitation_record_t IS 'Tenant invitation record table'; +COMMENT ON COLUMN nexent.tenant_invitation_record_t.invitation_record_id IS 'Invitation record ID, primary key'; +COMMENT ON COLUMN nexent.tenant_invitation_record_t.invitation_id IS 'Invitation ID, foreign key'; +COMMENT ON COLUMN nexent.tenant_invitation_record_t.user_id IS 'User ID'; +COMMENT ON COLUMN nexent.tenant_invitation_record_t.create_time IS 'Create time'; +COMMENT ON COLUMN nexent.tenant_invitation_record_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.tenant_invitation_record_t.created_by IS 'Created by'; +COMMENT ON COLUMN nexent.tenant_invitation_record_t.updated_by IS 'Updated by'; +COMMENT ON COLUMN nexent.tenant_invitation_record_t.delete_flag IS 'Delete flag, Y/N'; + +-- 3. Create tenant_group_info_t table for group information +CREATE TABLE IF NOT EXISTS nexent.tenant_group_info_t ( + group_id SERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + group_name VARCHAR(100) NOT NULL, + group_description VARCHAR(500), + create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), + update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Add comments for tenant_group_info_t table +COMMENT ON TABLE nexent.tenant_group_info_t IS 'Tenant group information table'; +COMMENT ON COLUMN nexent.tenant_group_info_t.group_id IS 'Group ID, primary key'; +COMMENT ON COLUMN nexent.tenant_group_info_t.tenant_id IS 'Tenant ID, foreign key'; +COMMENT ON COLUMN nexent.tenant_group_info_t.group_name IS 'Group name'; +COMMENT ON COLUMN nexent.tenant_group_info_t.group_description IS 'Group description'; +COMMENT ON COLUMN nexent.tenant_group_info_t.create_time IS 'Create time'; +COMMENT ON COLUMN nexent.tenant_group_info_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.tenant_group_info_t.created_by IS 'Created by'; +COMMENT ON COLUMN nexent.tenant_group_info_t.updated_by IS 'Updated by'; +COMMENT ON COLUMN nexent.tenant_group_info_t.delete_flag IS 'Delete flag, Y/N'; + +-- 4. Create tenant_group_user_t table for group user membership +CREATE TABLE IF NOT EXISTS nexent.tenant_group_user_t ( + group_user_id SERIAL PRIMARY KEY, + group_id INT4 NOT NULL, + user_id VARCHAR(100) NOT NULL, + create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), + update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Add comments for tenant_group_user_t table +COMMENT ON TABLE nexent.tenant_group_user_t IS 'Tenant group user membership table'; +COMMENT ON COLUMN nexent.tenant_group_user_t.group_user_id IS 'Group user ID, primary key'; +COMMENT ON COLUMN nexent.tenant_group_user_t.group_id IS 'Group ID, foreign key'; +COMMENT ON COLUMN nexent.tenant_group_user_t.user_id IS 'User ID, foreign key'; +COMMENT ON COLUMN nexent.tenant_group_user_t.create_time IS 'Create time'; +COMMENT ON COLUMN nexent.tenant_group_user_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.tenant_group_user_t.created_by IS 'Created by'; +COMMENT ON COLUMN nexent.tenant_group_user_t.updated_by IS 'Updated by'; +COMMENT ON COLUMN nexent.tenant_group_user_t.delete_flag IS 'Delete flag, Y/N'; + +-- 5. Add fields to user_tenant_t table +ALTER TABLE nexent.user_tenant_t +ADD COLUMN IF NOT EXISTS user_role VARCHAR(30); + +-- Add comments for new fields in user_tenant_t table +COMMENT ON COLUMN nexent.user_tenant_t.user_role IS 'User role: SU, ADMIN, DEV, USER'; + +-- 6. Create role_permission_t table for role permissions +CREATE TABLE IF NOT EXISTS nexent.role_permission_t ( + role_permission_id SERIAL PRIMARY KEY, + user_role VARCHAR(30) NOT NULL, + permission_category VARCHAR(30), + permission_type VARCHAR(30), + permission_subtype VARCHAR(30) +); + +-- Add comments for role_permission_t table +COMMENT ON TABLE nexent.role_permission_t IS 'Role permission configuration table'; +COMMENT ON COLUMN nexent.role_permission_t.role_permission_id IS 'Role permission ID, primary key'; +COMMENT ON COLUMN nexent.role_permission_t.user_role IS 'User role: SU, ADMIN, DEV, USER'; +COMMENT ON COLUMN nexent.role_permission_t.permission_category IS 'Permission category'; +COMMENT ON COLUMN nexent.role_permission_t.permission_type IS 'Permission type'; +COMMENT ON COLUMN nexent.role_permission_t.permission_subtype IS 'Permission subtype'; + +-- 7. Add fields to knowledge_record_t table +ALTER TABLE nexent.knowledge_record_t +ADD COLUMN IF NOT EXISTS group_ids VARCHAR, -- int4 list +ADD COLUMN IF NOT EXISTS ingroup_permission VARCHAR(30); + +-- Add comments for new fields in knowledge_record_t table +COMMENT ON COLUMN nexent.knowledge_record_t.group_ids IS 'Knowledge base group IDs list'; +COMMENT ON COLUMN nexent.knowledge_record_t.ingroup_permission IS 'In-group permission: EDIT, READ_ONLY, PRIVATE'; + +-- 8. Add fields to ag_tenant_agent_t table +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS group_ids VARCHAR; -- int4 list + +-- Add comments for new fields in ag_tenant_agent_t table +COMMENT ON COLUMN nexent.ag_tenant_agent_t.group_ids IS 'Agent group IDs list'; + +-- 9. Insert role permission data +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES +(1, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(2, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), +(3, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), +(4, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), +(5, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), +(6, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), +(7, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), +(8, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), +(9, 'SU', 'RESOURCE', 'AGENT', 'READ'), +(10, 'SU', 'RESOURCE', 'AGENT', 'DELETE'), +(11, 'SU', 'RESOURCE', 'KB', 'READ'), +(12, 'SU', 'RESOURCE', 'KB', 'DELETE'), +(13, 'SU', 'RESOURCE', 'KB.GROUPS', 'READ'), +(14, 'SU', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), +(15, 'SU', 'RESOURCE', 'KB.GROUPS', 'DELETE'), +(16, 'SU', 'RESOURCE', 'USER.ROLE', 'READ'), +(17, 'SU', 'RESOURCE', 'USER.ROLE', 'UPDATE'), +(18, 'SU', 'RESOURCE', 'USER.ROLE', 'DELETE'), +(19, 'SU', 'RESOURCE', 'MCP', 'READ'), +(20, 'SU', 'RESOURCE', 'MCP', 'DELETE'), +(21, 'SU', 'RESOURCE', 'MEM.SETTING', 'READ'), +(22, 'SU', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), +(23, 'SU', 'RESOURCE', 'MEM.AGENT', 'READ'), +(24, 'SU', 'RESOURCE', 'MEM.AGENT', 'DELETE'), +(25, 'SU', 'RESOURCE', 'MEM.PRIVATE', 'READ'), +(26, 'SU', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), +(27, 'SU', 'RESOURCE', 'MODEL', 'CREATE'), +(28, 'SU', 'RESOURCE', 'MODEL', 'READ'), +(29, 'SU', 'RESOURCE', 'MODEL', 'UPDATE'), +(30, 'SU', 'RESOURCE', 'MODEL', 'DELETE'), +(31, 'SU', 'RESOURCE', 'TENANT', 'CREATE'), +(32, 'SU', 'RESOURCE', 'TENANT', 'READ'), +(33, 'SU', 'RESOURCE', 'TENANT', 'UPDATE'), +(34, 'SU', 'RESOURCE', 'TENANT', 'DELETE'), +(35, 'SU', 'RESOURCE', 'TENANT.INFO', 'READ'), +(36, 'SU', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), +(37, 'SU', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), +(38, 'SU', 'RESOURCE', 'TENANT.INVITE', 'READ'), +(39, 'SU', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), +(40, 'SU', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), +(41, 'SU', 'RESOURCE', 'GROUP', 'CREATE'), +(42, 'SU', 'RESOURCE', 'GROUP', 'READ'), +(43, 'SU', 'RESOURCE', 'GROUP', 'UPDATE'), +(44, 'SU', 'RESOURCE', 'GROUP', 'DELETE'), +(45, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(46, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(47, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), +(48, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), +(49, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), +(50, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), +(51, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), +(52, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), +(53, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), +(54, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), +(55, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), +(56, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), +(57, 'ADMIN', 'RESOURCE', 'AGENT', 'CREATE'), +(58, 'ADMIN', 'RESOURCE', 'AGENT', 'READ'), +(59, 'ADMIN', 'RESOURCE', 'AGENT', 'UPDATE'), +(60, 'ADMIN', 'RESOURCE', 'AGENT', 'DELETE'), +(61, 'ADMIN', 'RESOURCE', 'KB', 'CREATE'), +(62, 'ADMIN', 'RESOURCE', 'KB', 'READ'), +(63, 'ADMIN', 'RESOURCE', 'KB', 'UPDATE'), +(64, 'ADMIN', 'RESOURCE', 'KB', 'DELETE'), +(65, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'READ'), +(66, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), +(67, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'DELETE'), +(68, 'ADMIN', 'RESOURCE', 'USER.ROLE', 'READ'), +(69, 'ADMIN', 'RESOURCE', 'MCP', 'CREATE'), +(70, 'ADMIN', 'RESOURCE', 'MCP', 'READ'), +(71, 'ADMIN', 'RESOURCE', 'MCP', 'UPDATE'), +(72, 'ADMIN', 'RESOURCE', 'MCP', 'DELETE'), +(73, 'ADMIN', 'RESOURCE', 'MEM.SETTING', 'READ'), +(74, 'ADMIN', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), +(75, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'CREATE'), +(76, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'READ'), +(77, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'DELETE'), +(78, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), +(79, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'READ'), +(80, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), +(81, 'ADMIN', 'RESOURCE', 'MODEL', 'CREATE'), +(82, 'ADMIN', 'RESOURCE', 'MODEL', 'READ'), +(83, 'ADMIN', 'RESOURCE', 'MODEL', 'UPDATE'), +(84, 'ADMIN', 'RESOURCE', 'MODEL', 'DELETE'), +(85, 'ADMIN', 'RESOURCE', 'TENANT.INFO', 'READ'), +(86, 'ADMIN', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), +(87, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), +(88, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'READ'), +(89, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), +(90, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), +(91, 'ADMIN', 'RESOURCE', 'GROUP', 'CREATE'), +(92, 'ADMIN', 'RESOURCE', 'GROUP', 'READ'), +(93, 'ADMIN', 'RESOURCE', 'GROUP', 'UPDATE'), +(94, 'ADMIN', 'RESOURCE', 'GROUP', 'DELETE'), +(95, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(96, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(97, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), +(98, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), +(99, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), +(100, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), +(101, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), +(102, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), +(103, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), +(104, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), +(105, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), +(106, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), +(107, 'DEV', 'RESOURCE', 'AGENT', 'CREATE'), +(108, 'DEV', 'RESOURCE', 'AGENT', 'READ'), +(109, 'DEV', 'RESOURCE', 'AGENT', 'UPDATE'), +(110, 'DEV', 'RESOURCE', 'AGENT', 'DELETE'), +(111, 'DEV', 'RESOURCE', 'KB', 'CREATE'), +(112, 'DEV', 'RESOURCE', 'KB', 'READ'), +(113, 'DEV', 'RESOURCE', 'KB', 'UPDATE'), +(114, 'DEV', 'RESOURCE', 'KB', 'DELETE'), +(115, 'DEV', 'RESOURCE', 'KB.GROUPS', 'READ'), +(116, 'DEV', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), +(117, 'DEV', 'RESOURCE', 'KB.GROUPS', 'DELETE'), +(118, 'DEV', 'RESOURCE', 'USER.ROLE', 'READ'), +(119, 'DEV', 'RESOURCE', 'MCP', 'CREATE'), +(120, 'DEV', 'RESOURCE', 'MCP', 'READ'), +(121, 'DEV', 'RESOURCE', 'MCP', 'UPDATE'), +(122, 'DEV', 'RESOURCE', 'MCP', 'DELETE'), +(123, 'DEV', 'RESOURCE', 'MEM.SETTING', 'READ'), +(124, 'DEV', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), +(125, 'DEV', 'RESOURCE', 'MEM.AGENT', 'READ'), +(126, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), +(127, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'READ'), +(128, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), +(129, 'DEV', 'RESOURCE', 'MODEL', 'READ'), +(130, 'DEV', 'RESOURCE', 'TENANT.INFO', 'READ'), +(131, 'DEV', 'RESOURCE', 'GROUP', 'READ'), +(132, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(133, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(134, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), +(135, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), +(136, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), +(137, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), +(138, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), +(139, 'USER', 'RESOURCE', 'AGENT', 'READ'), +(140, 'USER', 'RESOURCE', 'KB', 'CREATE'), +(141, 'USER', 'RESOURCE', 'KB', 'READ'), +(142, 'USER', 'RESOURCE', 'KB', 'UPDATE'), +(143, 'USER', 'RESOURCE', 'KB', 'DELETE'), +(144, 'USER', 'RESOURCE', 'KB.GROUPS', 'READ'), +(145, 'USER', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), +(146, 'USER', 'RESOURCE', 'KB.GROUPS', 'DELETE'), +(147, 'USER', 'RESOURCE', 'USER.ROLE', 'READ'), +(148, 'USER', 'RESOURCE', 'MCP', 'CREATE'), +(149, 'USER', 'RESOURCE', 'MCP', 'READ'), +(150, 'USER', 'RESOURCE', 'MCP', 'UPDATE'), +(151, 'USER', 'RESOURCE', 'MCP', 'DELETE'), +(152, 'USER', 'RESOURCE', 'MEM.SETTING', 'READ'), +(153, 'USER', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), +(154, 'USER', 'RESOURCE', 'MEM.AGENT', 'READ'), +(155, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), +(156, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'READ'), +(157, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), +(158, 'USER', 'RESOURCE', 'MODEL', 'READ'), +(159, 'USER', 'RESOURCE', 'TENANT.INFO', 'READ'), +(160, 'USER', 'RESOURCE', 'GROUP', 'READ'), +(161, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(162, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(163, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), +(164, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), +(165, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), +(166, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), +(167, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), +(168, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), +(169, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), +(170, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), +(171, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), +(172, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), +(173, 'SPEED', 'RESOURCE', 'AGENT', 'CREATE'), +(174, 'SPEED', 'RESOURCE', 'AGENT', 'READ'), +(175, 'SPEED', 'RESOURCE', 'AGENT', 'UPDATE'), +(176, 'SPEED', 'RESOURCE', 'AGENT', 'DELETE'), +(177, 'SPEED', 'RESOURCE', 'KB', 'CREATE'), +(178, 'SPEED', 'RESOURCE', 'KB', 'READ'), +(179, 'SPEED', 'RESOURCE', 'KB', 'UPDATE'), +(180, 'SPEED', 'RESOURCE', 'KB', 'DELETE'), +(181, 'SPEED', 'RESOURCE', 'KB.GROUPS', 'READ'), +(182, 'SPEED', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), +(183, 'SPEED', 'RESOURCE', 'KB.GROUPS', 'DELETE'), +(184, 'SPEED', 'RESOURCE', 'USER.ROLE', 'READ'), +(185, 'SPEED', 'RESOURCE', 'MCP', 'CREATE'), +(186, 'SPEED', 'RESOURCE', 'MCP', 'READ'), +(187, 'SPEED', 'RESOURCE', 'MCP', 'UPDATE'), +(188, 'SPEED', 'RESOURCE', 'MCP', 'DELETE'), +(189, 'SPEED', 'RESOURCE', 'MEM.SETTING', 'READ'), +(190, 'SPEED', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), +(191, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'CREATE'), +(192, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'READ'), +(193, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'DELETE'), +(194, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), +(195, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'READ'), +(196, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), +(197, 'SPEED', 'RESOURCE', 'MODEL', 'CREATE'), +(198, 'SPEED', 'RESOURCE', 'MODEL', 'READ'), +(199, 'SPEED', 'RESOURCE', 'MODEL', 'UPDATE'), +(200, 'SPEED', 'RESOURCE', 'MODEL', 'DELETE'), +(201, 'SPEED', 'RESOURCE', 'TENANT.INFO', 'READ'), +(202, 'SPEED', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), +(203, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), +(204, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'READ'), +(205, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), +(206, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), +(207, 'SPEED', 'RESOURCE', 'GROUP', 'CREATE'), +(208, 'SPEED', 'RESOURCE', 'GROUP', 'READ'), +(209, 'SPEED', 'RESOURCE', 'GROUP', 'UPDATE'), +(210, 'SPEED', 'RESOURCE', 'GROUP', 'DELETE') +ON CONFLICT (role_permission_id) DO NOTHING; + +-- Add is_new column to ag_tenant_agent_t table for new agent marking +-- This migration adds a field to track whether an agent is marked as new for users + +-- Add is_new column with default value false +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS is_new BOOLEAN DEFAULT FALSE; + +-- Add comment for the new column +COMMENT ON COLUMN nexent.ag_tenant_agent_t.is_new IS 'Whether this agent is marked as new for the user'; + +-- Create index for performance on is_new queries +CREATE INDEX IF NOT EXISTS idx_ag_tenant_agent_t_is_new +ON nexent.ag_tenant_agent_t (tenant_id, is_new) +WHERE delete_flag = 'N'; + + + +-- Add user_email column to user_tenant_t table +ALTER TABLE nexent.user_tenant_t +ADD COLUMN IF NOT EXISTS user_email VARCHAR(255); + +-- Add comment to the new column +COMMENT ON COLUMN nexent.user_tenant_t.user_email IS 'User email address'; + +INSERT INTO nexent.user_tenant_t (user_id, tenant_id, user_role, user_email, created_by, updated_by) +VALUES ('user_id', 'tenant_id', 'SPEED', NULL, 'system', 'system') +ON CONFLICT (user_id, tenant_id) DO NOTHING; + +ALTER TABLE nexent.mcp_record_t +ADD COLUMN IF NOT EXISTS container_id VARCHAR(200); + +COMMENT ON COLUMN nexent.mcp_record_t.container_id IS 'Docker container ID for MCP service, NULL for non-containerized MCP'; + + + +CREATE SEQUENCE IF NOT EXISTS "nexent"."ag_tenant_agent_t_agent_id_seq" +INCREMENT 1 +MINVALUE 1 +MAXVALUE 2147483647 +START 1 +CACHE 1; + +-- Delete erroneous tenant with empty tenant_id and all related data +-- This script removes records where tenant_id is empty string from tenant_config_t and tenant_group_info_t + +-- 1. Force delete all records in tenant_config_t where tenant_id is empty string +DELETE FROM nexent.tenant_config_t +WHERE tenant_id = ''; + +-- 2. Force delete all records in tenant_group_info_t where tenant_id is empty string +DELETE FROM nexent.tenant_group_info_t +WHERE tenant_id = ''; + +-- Migration: Add authorization_token column to mcp_record_t table +-- Date: 2025-03-01 +-- Description: Add authorization_token field to support MCP server authentication + +-- Add authorization_token column to mcp_record_t table +ALTER TABLE nexent.mcp_record_t +ADD COLUMN IF NOT EXISTS authorization_token VARCHAR(500) DEFAULT NULL; + +-- Add comment to the column +COMMENT ON COLUMN nexent.mcp_record_t.authorization_token IS 'Authorization token for MCP server authentication (e.g., Bearer token)'; + +-- Migration: Add ingroup_permission column to ag_tenant_agent_t table +-- Date: 2025-03-02 +-- Description: Add ingroup_permission field to support in-group permission control for agents + +-- Add ingroup_permission column to ag_tenant_agent_t table +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS ingroup_permission VARCHAR(30) DEFAULT NULL; + +-- Add comment to the column +COMMENT ON COLUMN nexent.ag_tenant_agent_t.ingroup_permission IS 'In-group permission: EDIT, READ_ONLY, PRIVATE'; + +-- Step 1: Create sequence for auto-increment +CREATE SEQUENCE IF NOT EXISTS "nexent"."ag_tool_instance_t_tool_instance_id_seq" +INCREMENT 1 +MINVALUE 1 +MAXVALUE 2147483647 +START 1 +CACHE 1; + +CREATE SEQUENCE IF NOT EXISTS "nexent"."ag_agent_relation_t_relation_id_seq" +INCREMENT 1 +MINVALUE 1 +MAXVALUE 2147483647 +START 1 +CACHE 1; + +-- Initialize tenant group and default configuration for existing tenants +-- This migration adds default group and basic config for tenants that lack them +-- Trigger condition: tenant has no TENANT_ID config_key in tenant_config_t + +DO $$ +DECLARE + target_tenant_id VARCHAR(100); + new_group_id INTEGER; +BEGIN + -- Loop through each distinct tenant_id from user_tenant_t + FOR target_tenant_id IN + SELECT DISTINCT tenant_id + FROM nexent.user_tenant_t + WHERE tenant_id IS NOT NULL + LOOP + -- Check if tenant already has TENANT_ID config_key + IF NOT EXISTS ( + SELECT 1 FROM nexent.tenant_config_t + WHERE tenant_id = target_tenant_id + AND config_key = 'TENANT_ID' + AND delete_flag = 'N' + ) THEN + -- Insert TENANT_ID config + INSERT INTO nexent.tenant_config_t ( + tenant_id, user_id, value_type, config_key, config_value, + create_time, update_time, created_by, updated_by, delete_flag + ) VALUES ( + target_tenant_id, NULL, 'single', 'TENANT_ID', target_tenant_id, + NOW(), NOW(), 'system', 'system', 'N' + ); + + -- Insert TENANT_NAME config if not exists + IF NOT EXISTS ( + SELECT 1 FROM nexent.tenant_config_t + WHERE tenant_id = target_tenant_id + AND config_key = 'TENANT_NAME' + AND delete_flag = 'N' + ) THEN + INSERT INTO nexent.tenant_config_t ( + tenant_id, user_id, value_type, config_key, config_value, + create_time, update_time, created_by, updated_by, delete_flag + ) VALUES ( + target_tenant_id, NULL, 'single', 'TENANT_NAME', 'Unnamed Tenant', + NOW(), NOW(), 'system', 'system', 'N' + ); + END IF; + + -- Check if tenant already has a group + IF NOT EXISTS ( + SELECT 1 FROM nexent.tenant_group_info_t + WHERE tenant_id = target_tenant_id + AND delete_flag = 'N' + ) THEN + -- Insert default group + INSERT INTO nexent.tenant_group_info_t ( + tenant_id, group_name, group_description, + create_time, update_time, created_by, updated_by, delete_flag + ) VALUES ( + target_tenant_id, 'Default Group', 'Default group for tenant', + NOW(), NOW(), 'system', 'system', 'N' + ) RETURNING group_id INTO new_group_id; + + -- Insert DEFAULT_GROUP_ID config + IF new_group_id IS NOT NULL THEN + INSERT INTO nexent.tenant_config_t ( + tenant_id, user_id, value_type, config_key, config_value, + create_time, update_time, created_by, updated_by, delete_flag + ) VALUES ( + target_tenant_id, NULL, 'single', 'DEFAULT_GROUP_ID', new_group_id::VARCHAR, + NOW(), NOW(), 'system', 'system', 'N' + ); + END IF; + END IF; + END IF; + END LOOP; +END $$; + +-- 步骤 1：添�?nullable �?version_no 字段（不设默认值，让显式赋值） +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS version_no INTEGER NULL; + +ALTER TABLE nexent.ag_tool_instance_t +ADD COLUMN IF NOT EXISTS version_no INTEGER NULL; + +ALTER TABLE nexent.ag_agent_relation_t +ADD COLUMN IF NOT EXISTS version_no INTEGER NULL; + +-- 步骤 2：更新所有历史数据的 version_no �?0 +UPDATE nexent.ag_tenant_agent_t SET version_no = 0 WHERE version_no IS NULL; +UPDATE nexent.ag_tool_instance_t SET version_no = 0 WHERE version_no IS NULL; +UPDATE nexent.ag_agent_relation_t SET version_no = 0 WHERE version_no IS NULL; + +-- 步骤 3：将字段设为 NOT NULL，并设置默认�?0 +ALTER TABLE nexent.ag_tenant_agent_t ALTER COLUMN version_no SET NOT NULL; +ALTER TABLE nexent.ag_tenant_agent_t ALTER COLUMN version_no SET DEFAULT 0; + +ALTER TABLE nexent.ag_tool_instance_t ALTER COLUMN version_no SET NOT NULL; +ALTER TABLE nexent.ag_tool_instance_t ALTER COLUMN version_no SET DEFAULT 0; + +ALTER TABLE nexent.ag_agent_relation_t ALTER COLUMN version_no SET NOT NULL; +ALTER TABLE nexent.ag_agent_relation_t ALTER COLUMN version_no SET DEFAULT 0; + +-- 步骤 4：为 ag_tenant_agent_t 添加 current_version_no 字段 +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS current_version_no INTEGER NULL; + +-- 步骤5：修改主�? +ALTER TABLE nexent.ag_tenant_agent_t DROP CONSTRAINT IF EXISTS ag_tenant_agent_t_pkey; +ALTER TABLE nexent.ag_tenant_agent_t ADD CONSTRAINT ag_tenant_agent_t_pkey PRIMARY KEY (agent_id, version_no); + +ALTER TABLE nexent.ag_tool_instance_t DROP CONSTRAINT IF EXISTS ag_tool_instance_t_pkey; +ALTER TABLE nexent.ag_tool_instance_t ADD CONSTRAINT ag_tool_instance_t_pkey PRIMARY KEY (tool_instance_id, version_no); + +ALTER TABLE nexent.ag_agent_relation_t DROP CONSTRAINT IF EXISTS ag_agent_relation_t_pkey; +ALTER TABLE nexent.ag_agent_relation_t ADD CONSTRAINT ag_agent_relation_t_pkey PRIMARY KEY (relation_id, version_no); + +-- 步骤6：新增agent版本管理�? +CREATE TABLE IF NOT EXISTS nexent.ag_tenant_agent_version_t ( + id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + agent_id INTEGER NOT NULL, + version_no INTEGER NOT NULL, + version_name VARCHAR(100), -- 用户自定义版本名�? + release_note TEXT, -- 发布备注 + + source_version_no INTEGER NULL, -- 来源版本号（回滚时记录） + source_type VARCHAR(30) NULL, -- 来源类型：NORMAL(正常发布) / ROLLBACK(回滚产生) + + status VARCHAR(30) DEFAULT 'RELEASED', -- 版本状态：RELEASED / DISABLED / ARCHIVED + + created_by VARCHAR(100) NOT NULL, + create_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + updated_by VARCHAR(100), + update_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE nexent.ag_tenant_agent_version_t OWNER TO "root"; + +-- 步骤 7：添加COMMENT +COMMENT ON COLUMN nexent.ag_tenant_agent_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.current_version_no IS 'Current published version number. NULL means no version published yet'; +COMMENT ON COLUMN nexent.ag_tool_instance_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; +COMMENT ON COLUMN nexent.ag_agent_relation_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; + +COMMENT ON TABLE nexent.ag_tenant_agent_version_t IS 'Agent version metadata table. Stores version info, release notes, and version lineage.'; + +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.id IS 'Primary key, auto-increment'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.tenant_id IS 'Tenant ID'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.agent_id IS 'Agent ID'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.version_no IS 'Version number, starts from 1. Does not include 0 (draft)'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.version_name IS 'User-defined version name for display (e.g., "Stable v2.1", "Hotfix-001"). NULL means use version_no as display.'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.release_note IS 'Release notes / publish remarks'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.source_version_no IS 'Source version number. If this version is a rollback, record the source version number.'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.source_type IS 'Source type: NORMAL (normal publish) / ROLLBACK (rollback and republish).'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.status IS 'Version status: RELEASED / DISABLED / ARCHIVED'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.created_by IS 'User who published this version'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.create_time IS 'Version creation timestamp'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.updated_by IS 'Last user who updated this version'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.update_time IS 'Last update timestamp'; +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.delete_flag IS 'Soft delete flag: Y/N'; + +DELETE FROM nexent.role_permission_t; + +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES +(1, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(2, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), +(3, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/tenant-resources'), +(4, 'SU', 'RESOURCE', 'AGENT', 'READ'), +(5, 'SU', 'RESOURCE', 'AGENT', 'DELETE'), +(6, 'SU', 'RESOURCE', 'KB', 'READ'), +(7, 'SU', 'RESOURCE', 'KB', 'DELETE'), +(8, 'SU', 'RESOURCE', 'KB.GROUPS', 'READ'), +(9, 'SU', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), +(10, 'SU', 'RESOURCE', 'KB.GROUPS', 'DELETE'), +(11, 'SU', 'RESOURCE', 'USER.ROLE', 'READ'), +(12, 'SU', 'RESOURCE', 'USER.ROLE', 'UPDATE'), +(13, 'SU', 'RESOURCE', 'USER.ROLE', 'DELETE'), +(14, 'SU', 'RESOURCE', 'MCP', 'READ'), +(15, 'SU', 'RESOURCE', 'MCP', 'DELETE'), +(16, 'SU', 'RESOURCE', 'MEM.SETTING', 'READ'), +(17, 'SU', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), +(18, 'SU', 'RESOURCE', 'MEM.AGENT', 'READ'), +(19, 'SU', 'RESOURCE', 'MEM.AGENT', 'DELETE'), +(20, 'SU', 'RESOURCE', 'MEM.PRIVATE', 'READ'), +(21, 'SU', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), +(22, 'SU', 'RESOURCE', 'MODEL', 'CREATE'), +(23, 'SU', 'RESOURCE', 'MODEL', 'READ'), +(24, 'SU', 'RESOURCE', 'MODEL', 'UPDATE'), +(25, 'SU', 'RESOURCE', 'MODEL', 'DELETE'), +(26, 'SU', 'RESOURCE', 'TENANT', 'CREATE'), +(27, 'SU', 'RESOURCE', 'TENANT', 'READ'), +(28, 'SU', 'RESOURCE', 'TENANT', 'UPDATE'), +(29, 'SU', 'RESOURCE', 'TENANT', 'DELETE'), +(30, 'SU', 'RESOURCE', 'TENANT.LIST', 'READ'), +(31, 'SU', 'RESOURCE', 'TENANT.INFO', 'READ'), +(32, 'SU', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), +(33, 'SU', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), +(34, 'SU', 'RESOURCE', 'TENANT.INVITE', 'READ'), +(35, 'SU', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), +(36, 'SU', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), +(37, 'SU', 'RESOURCE', 'GROUP', 'CREATE'), +(38, 'SU', 'RESOURCE', 'GROUP', 'READ'), +(39, 'SU', 'RESOURCE', 'GROUP', 'UPDATE'), +(40, 'SU', 'RESOURCE', 'GROUP', 'DELETE'), +(41, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(42, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(43, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), +(44, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), +(45, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), +(46, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), +(47, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), +(48, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), +(49, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), +(50, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), +(51, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), +(52, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), +(53, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/tenant-resources'), +(54, 'ADMIN', 'RESOURCE', 'AGENT', 'CREATE'), +(55, 'ADMIN', 'RESOURCE', 'AGENT', 'READ'), +(56, 'ADMIN', 'RESOURCE', 'AGENT', 'UPDATE'), +(57, 'ADMIN', 'RESOURCE', 'AGENT', 'DELETE'), +(58, 'ADMIN', 'RESOURCE', 'KB', 'CREATE'), +(59, 'ADMIN', 'RESOURCE', 'KB', 'READ'), +(60, 'ADMIN', 'RESOURCE', 'KB', 'UPDATE'), +(61, 'ADMIN', 'RESOURCE', 'KB', 'DELETE'), +(62, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'READ'), +(63, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), +(64, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'DELETE'), +(65, 'ADMIN', 'RESOURCE', 'USER.ROLE', 'READ'), +(66, 'ADMIN', 'RESOURCE', 'MCP', 'CREATE'), +(67, 'ADMIN', 'RESOURCE', 'MCP', 'READ'), +(68, 'ADMIN', 'RESOURCE', 'MCP', 'UPDATE'), +(69, 'ADMIN', 'RESOURCE', 'MCP', 'DELETE'), +(70, 'ADMIN', 'RESOURCE', 'MEM.SETTING', 'READ'), +(71, 'ADMIN', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), +(72, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'CREATE'), +(73, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'READ'), +(74, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'DELETE'), +(75, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), +(76, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'READ'), +(77, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), +(78, 'ADMIN', 'RESOURCE', 'MODEL', 'CREATE'), +(79, 'ADMIN', 'RESOURCE', 'MODEL', 'READ'), +(80, 'ADMIN', 'RESOURCE', 'MODEL', 'UPDATE'), +(81, 'ADMIN', 'RESOURCE', 'MODEL', 'DELETE'), +(82, 'ADMIN', 'RESOURCE', 'TENANT.INFO', 'READ'), +(83, 'ADMIN', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), +(84, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), +(85, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'READ'), +(86, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), +(87, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), +(88, 'ADMIN', 'RESOURCE', 'GROUP', 'CREATE'), +(89, 'ADMIN', 'RESOURCE', 'GROUP', 'READ'), +(90, 'ADMIN', 'RESOURCE', 'GROUP', 'UPDATE'), +(91, 'ADMIN', 'RESOURCE', 'GROUP', 'DELETE'), +(92, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(93, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(94, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), +(95, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), +(96, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), +(97, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), +(98, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), +(99, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), +(100, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), +(101, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), +(102, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), +(103, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), +(104, 'DEV', 'RESOURCE', 'AGENT', 'CREATE'), +(105, 'DEV', 'RESOURCE', 'AGENT', 'READ'), +(106, 'DEV', 'RESOURCE', 'AGENT', 'UPDATE'), +(107, 'DEV', 'RESOURCE', 'AGENT', 'DELETE'), +(108, 'DEV', 'RESOURCE', 'KB', 'CREATE'), +(109, 'DEV', 'RESOURCE', 'KB', 'READ'), +(110, 'DEV', 'RESOURCE', 'KB', 'UPDATE'), +(111, 'DEV', 'RESOURCE', 'KB', 'DELETE'), +(112, 'DEV', 'RESOURCE', 'KB.GROUPS', 'READ'), +(113, 'DEV', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), +(114, 'DEV', 'RESOURCE', 'KB.GROUPS', 'DELETE'), +(115, 'DEV', 'RESOURCE', 'USER.ROLE', 'READ'), +(116, 'DEV', 'RESOURCE', 'MCP', 'CREATE'), +(117, 'DEV', 'RESOURCE', 'MCP', 'READ'), +(118, 'DEV', 'RESOURCE', 'MCP', 'UPDATE'), +(119, 'DEV', 'RESOURCE', 'MCP', 'DELETE'), +(120, 'DEV', 'RESOURCE', 'MEM.SETTING', 'READ'), +(121, 'DEV', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), +(122, 'DEV', 'RESOURCE', 'MEM.AGENT', 'READ'), +(123, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), +(124, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'READ'), +(125, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), +(126, 'DEV', 'RESOURCE', 'MODEL', 'READ'), +(127, 'DEV', 'RESOURCE', 'TENANT.INFO', 'READ'), +(128, 'DEV', 'RESOURCE', 'GROUP', 'READ'), +(129, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(130, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(131, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), +(132, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), +(133, 'USER', 'RESOURCE', 'AGENT', 'READ'), +(134, 'USER', 'RESOURCE', 'USER.ROLE', 'READ'), +(135, 'USER', 'RESOURCE', 'MEM.SETTING', 'READ'), +(136, 'USER', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), +(137, 'USER', 'RESOURCE', 'MEM.AGENT', 'READ'), +(138, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), +(139, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'READ'), +(140, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), +(141, 'USER', 'RESOURCE', 'TENANT.INFO', 'READ'), +(142, 'USER', 'RESOURCE', 'GROUP', 'READ'), +(143, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(144, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(145, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), +(146, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), +(147, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), +(148, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), +(149, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), +(150, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), +(151, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), +(152, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), +(153, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), +(154, 'SPEED', 'RESOURCE', 'AGENT', 'CREATE'), +(155, 'SPEED', 'RESOURCE', 'AGENT', 'READ'), +(156, 'SPEED', 'RESOURCE', 'AGENT', 'UPDATE'), +(157, 'SPEED', 'RESOURCE', 'AGENT', 'DELETE'), +(158, 'SPEED', 'RESOURCE', 'KB', 'CREATE'), +(159, 'SPEED', 'RESOURCE', 'KB', 'READ'), +(160, 'SPEED', 'RESOURCE', 'KB', 'UPDATE'), +(161, 'SPEED', 'RESOURCE', 'KB', 'DELETE'), +(166, 'SPEED', 'RESOURCE', 'MCP', 'CREATE'), +(167, 'SPEED', 'RESOURCE', 'MCP', 'READ'), +(168, 'SPEED', 'RESOURCE', 'MCP', 'UPDATE'), +(169, 'SPEED', 'RESOURCE', 'MCP', 'DELETE'), +(170, 'SPEED', 'RESOURCE', 'MEM.SETTING', 'READ'), +(171, 'SPEED', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), +(172, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'CREATE'), +(173, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'READ'), +(174, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'DELETE'), +(175, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), +(176, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'READ'), +(177, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), +(178, 'SPEED', 'RESOURCE', 'MODEL', 'CREATE'), +(179, 'SPEED', 'RESOURCE', 'MODEL', 'READ'), +(180, 'SPEED', 'RESOURCE', 'MODEL', 'UPDATE'), +(181, 'SPEED', 'RESOURCE', 'MODEL', 'DELETE'), +(182, 'SPEED', 'RESOURCE', 'TENANT.INFO', 'READ'), +(183, 'SPEED', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), +(184, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), +(185, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'READ'), +(186, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), +(187, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'DELETE') +ON CONFLICT (role_permission_id) DO NOTHING; + +-- Migration: Add user_token_info_t and user_token_usage_log_t tables +-- Date: 2026-03-06 +-- Description: Create user token (AK/SK) management tables with audit fields + +-- Set search path to nexent schema +SET search_path TO nexent; + +-- Create the user_token_info_t table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.user_token_info_t ( + token_id SERIAL4 PRIMARY KEY NOT NULL, + access_key VARCHAR(100) NOT NULL, + user_id VARCHAR(100) NOT NULL, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE "user_token_info_t" OWNER TO "root"; + +-- Add comment to the table +COMMENT ON TABLE nexent.user_token_info_t IS 'User token (AK/SK) information table'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.user_token_info_t.token_id IS 'Token ID, unique primary key'; +COMMENT ON COLUMN nexent.user_token_info_t.access_key IS 'Access Key (AK)'; +COMMENT ON COLUMN nexent.user_token_info_t.user_id IS 'User ID who owns this token'; +COMMENT ON COLUMN nexent.user_token_info_t.create_time IS 'Creation time, audit field'; +COMMENT ON COLUMN nexent.user_token_info_t.update_time IS 'Update time, audit field'; +COMMENT ON COLUMN nexent.user_token_info_t.created_by IS 'Creator ID, audit field'; +COMMENT ON COLUMN nexent.user_token_info_t.updated_by IS 'Last updater ID, audit field'; +COMMENT ON COLUMN nexent.user_token_info_t.delete_flag IS 'Soft delete flag, Y means deleted'; + + +-- Create the user_token_usage_log_t table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.user_token_usage_log_t ( + token_usage_id SERIAL4 PRIMARY KEY NOT NULL, + token_id INT4 NOT NULL, + call_function_name VARCHAR(100), + related_id INT4, + meta_data JSONB, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE "user_token_usage_log_t" OWNER TO "root"; + +-- Add comment to the table +COMMENT ON TABLE nexent.user_token_usage_log_t IS 'User token usage log table'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.user_token_usage_log_t.token_usage_id IS 'Token usage log ID, unique primary key'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.token_id IS 'Foreign key to user_token_info_t.token_id'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.call_function_name IS 'API function name being called'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.related_id IS 'Related resource ID (e.g., conversation_id)'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.meta_data IS 'Additional metadata for this usage log entry, stored as JSON'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.create_time IS 'Creation time, audit field'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.update_time IS 'Update time, audit field'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.created_by IS 'Creator ID, audit field'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.updated_by IS 'Last updater ID, audit field'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.delete_flag IS 'Soft delete flag, Y means deleted'; + +-- Migration: Remove partner_mapping_id_t table for northbound conversation ID mapping +-- Date: 2026-03-10 +-- Description: Remove the external-internal conversation ID mapping table as northbound APIs now use internal conversation IDs directly +-- Note: This table is no longer needed after refactoring northbound authentication logic + +-- Drop the partner_mapping_id_t table if it exists +DROP TABLE IF EXISTS nexent.partner_mapping_id_t CASCADE; + +-- Drop the associated sequence if it exists +DROP SEQUENCE IF EXISTS nexent.partner_mapping_id_t_id_seq; diff --git a/docker/sql/v2.0.2_0414_add_a2a_tables.sql b/deploy/sql/migrations/v2.0_merged_migrations.sql similarity index 53% rename from docker/sql/v2.0.2_0414_add_a2a_tables.sql rename to deploy/sql/migrations/v2.0_merged_migrations.sql index 8b3c3e3c9..ea3b0d421 100644 --- a/docker/sql/v2.0.2_0414_add_a2a_tables.sql +++ b/deploy/sql/migrations/v2.0_merged_migrations.sql @@ -1,3 +1,203 @@ +-- Nexent merged SQL migrations: v2.0 +-- This file is generated from historical migration files. + +-- Migration: Add ag_skill_info_t, ag_skill_tools_rel_t, and ag_skill_instance_t tables +-- Date: 2026-03-14 +-- Description: Create skill management tables with skill content, tags, and tool relationships + +SET search_path TO nexent; + +-- Create the ag_skill_info_t table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.ag_skill_info_t ( + skill_id SERIAL4 PRIMARY KEY NOT NULL, + skill_name VARCHAR(100) NOT NULL, + skill_description VARCHAR(1000), + skill_tags JSON, + skill_content TEXT, + params JSON, + source VARCHAR(30) DEFAULT 'official', + created_by VARCHAR(100), + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_by VARCHAR(100), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE "ag_skill_info_t" OWNER TO "root"; + +-- Add comment to the table +COMMENT ON TABLE nexent.ag_skill_info_t IS 'Skill information table for managing custom skills'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.ag_skill_info_t.skill_id IS 'Skill ID, unique primary key'; +COMMENT ON COLUMN nexent.ag_skill_info_t.skill_name IS 'Skill name, globally unique'; +COMMENT ON COLUMN nexent.ag_skill_info_t.skill_description IS 'Skill description text'; +COMMENT ON COLUMN nexent.ag_skill_info_t.skill_tags IS 'Skill tags stored as JSON array'; +COMMENT ON COLUMN nexent.ag_skill_info_t.skill_content IS 'Skill content or prompt text'; +COMMENT ON COLUMN nexent.ag_skill_info_t.source IS 'Skill source: official, custom, or partner'; +COMMENT ON COLUMN nexent.ag_skill_info_t.created_by IS 'Creator ID'; +COMMENT ON COLUMN nexent.ag_skill_info_t.create_time IS 'Creation timestamp'; +COMMENT ON COLUMN nexent.ag_skill_info_t.updated_by IS 'Last updater ID'; +COMMENT ON COLUMN nexent.ag_skill_info_t.update_time IS 'Last update timestamp'; +COMMENT ON COLUMN nexent.ag_skill_info_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; + +-- Create the ag_skill_tools_rel_t table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.ag_skill_tools_rel_t ( + rel_id SERIAL4 PRIMARY KEY NOT NULL, + skill_id INTEGER, + tool_id INTEGER, + created_by VARCHAR(100), + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_by VARCHAR(100), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE "ag_skill_tools_rel_t" OWNER TO "root"; + +-- Add comment to the table +COMMENT ON TABLE nexent.ag_skill_tools_rel_t IS 'Skill-tool relationship table for many-to-many mapping'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.rel_id IS 'Relationship ID, unique primary key'; +COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.skill_id IS 'Foreign key to ag_skill_info_t.skill_id'; +COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.tool_id IS 'Tool ID from ag_tool_info_t'; +COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.created_by IS 'Creator ID'; +COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.create_time IS 'Creation timestamp'; +COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.updated_by IS 'Last updater ID'; +COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.update_time IS 'Last update timestamp'; +COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; + +-- Create the ag_skill_instance_t table in the nexent schema +-- Stores skill instance configuration per agent version +-- Note: skill_description and skill_content fields removed, now retrieved from ag_skill_info_t +CREATE TABLE IF NOT EXISTS nexent.ag_skill_instance_t ( + skill_instance_id SERIAL4 NOT NULL, + skill_id INTEGER NOT NULL, + agent_id INTEGER NOT NULL, + user_id VARCHAR(100), + tenant_id VARCHAR(100), + enabled BOOLEAN DEFAULT TRUE, + version_no INTEGER DEFAULT 0 NOT NULL, + created_by VARCHAR(100), + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_by VARCHAR(100), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + delete_flag VARCHAR(1) DEFAULT 'N', + CONSTRAINT ag_skill_instance_t_pkey PRIMARY KEY (skill_instance_id, version_no) +); + +ALTER TABLE "ag_skill_instance_t" OWNER TO "root"; + +-- Add comment to the table +COMMENT ON TABLE nexent.ag_skill_instance_t IS 'Skill instance configuration table - stores per-agent skill settings'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.ag_skill_instance_t.skill_instance_id IS 'Skill instance ID'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.skill_id IS 'Foreign key to ag_skill_info_t.skill_id'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.agent_id IS 'Agent ID'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.user_id IS 'User ID'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.tenant_id IS 'Tenant ID'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.enabled IS 'Whether this skill is enabled for the agent'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.created_by IS 'Creator ID'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.create_time IS 'Creation timestamp'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.updated_by IS 'Last updater ID'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.update_time IS 'Last update timestamp'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; + +-- v2.0.1_0331_add_outer_api_tool_t.sql +-- Create table for outer API tools (OpenAPI to MCP conversion) + +-- Create the ag_outer_api_tools table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.ag_outer_api_tools ( + id BIGSERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL, + description TEXT, + method VARCHAR(10), + url TEXT NOT NULL, + headers_template JSONB DEFAULT '{}', + query_template JSONB DEFAULT '{}', + body_template JSONB DEFAULT '{}', + input_schema JSONB DEFAULT '{}', + tenant_id VARCHAR(100), + is_available BOOLEAN DEFAULT TRUE, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE nexent.ag_outer_api_tools OWNER TO "root"; + +-- Create a function to update the update_time column +CREATE OR REPLACE FUNCTION update_ag_outer_api_tools_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Create a trigger to call the function before each update +DROP TRIGGER IF EXISTS update_ag_outer_api_tools_update_time_trigger ON nexent.ag_outer_api_tools; +CREATE TRIGGER update_ag_outer_api_tools_update_time_trigger +BEFORE UPDATE ON nexent.ag_outer_api_tools +FOR EACH ROW +EXECUTE FUNCTION update_ag_outer_api_tools_update_time(); + +-- Add comment to the table +COMMENT ON TABLE nexent.ag_outer_api_tools IS 'Outer API tools table - stores converted OpenAPI tools as MCP tools'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.ag_outer_api_tools.id IS 'Tool ID, unique primary key'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.name IS 'Tool name (unique identifier)'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.description IS 'Tool description'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.method IS 'HTTP method: GET/POST/PUT/DELETE/PATCH'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.url IS 'API endpoint URL (full path with base URL)'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.headers_template IS 'Headers template as JSONB'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.query_template IS 'Query parameters template as JSONB'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.body_template IS 'Request body template as JSONB'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.input_schema IS 'MCP input schema as JSONB'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.tenant_id IS 'Tenant ID for multi-tenancy'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.is_available IS 'Whether the tool is available'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.created_by IS 'Creator'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.updated_by IS 'Updater'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; + +-- Create index for tenant_id queries +CREATE INDEX IF NOT EXISTS idx_ag_outer_api_tools_tenant_id +ON nexent.ag_outer_api_tools (tenant_id) +WHERE delete_flag = 'N'; + +-- Create index for name queries +CREATE INDEX IF NOT EXISTS idx_ag_outer_api_tools_name +ON nexent.ag_outer_api_tools (name) +WHERE delete_flag = 'N'; + +-- v2.0.2_0410_add_columns_outer_api_tools.sql +-- Add MCP service-level columns to ag_outer_api_tools table +-- These columns enable grouping tools from the same OpenAPI spec under a single MCP service + +-- Add columns for MCP service information +ALTER TABLE nexent.ag_outer_api_tools + ADD COLUMN IF NOT EXISTS mcp_service_name VARCHAR(100), + ADD COLUMN IF NOT EXISTS openapi_json JSONB, + ADD COLUMN IF NOT EXISTS server_url VARCHAR(500); + +-- Add comments to the new columns +COMMENT ON COLUMN nexent.ag_outer_api_tools.mcp_service_name IS 'MCP service name for grouping tools from same OpenAPI spec'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.openapi_json IS 'Complete OpenAPI JSON specification'; +COMMENT ON COLUMN nexent.ag_outer_api_tools.server_url IS 'Base URL of the REST API server'; + +-- Create index for mcp_service_name queries +CREATE INDEX IF NOT EXISTS idx_ag_outer_api_tools_mcp_service_name +ON nexent.ag_outer_api_tools (mcp_service_name) +WHERE delete_flag = 'N' AND mcp_service_name IS NOT NULL; + -- A2A Protocol Tables Migration -- Purpose: Support A2A (Agent-to-Agent) protocol with both Client (discover and call external agents) and Server (expose local agents) capabilities -- Tables created: @@ -418,3 +618,245 @@ COMMENT ON COLUMN nexent.ag_a2a_artifact_t.parts IS 'Artifact parts following A2 COMMENT ON COLUMN nexent.ag_a2a_artifact_t.meta_data IS 'Artifact metadata'; COMMENT ON COLUMN nexent.ag_a2a_artifact_t.extensions IS 'Extension URI list'; COMMENT ON COLUMN nexent.ag_a2a_artifact_t.create_time IS 'Artifact creation timestamp'; + +-- Migration: Convert ag_outer_api_tools (tool-level) to ag_outer_api_services (service-level) +-- Date: 2026-04-09 +-- Description: Each OpenAPI service now stores one record instead of one record per tool. +-- Only service-level fields (mcp_service_name, openapi_json, server_url, etc.) are kept. + +-- Step 1: Create new table for services +CREATE TABLE IF NOT EXISTS nexent.ag_outer_api_services ( + id BIGSERIAL PRIMARY KEY, + mcp_service_name VARCHAR(100) NOT NULL, + description TEXT, + openapi_json JSONB, + server_url VARCHAR(500), + headers_template JSONB, + tenant_id VARCHAR(100) NOT NULL, + is_available BOOLEAN DEFAULT TRUE, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Step 2: Migrate data - one record per service +-- Use DISTINCT ON to get one record per (tenant_id, mcp_service_name) +-- Order by update_time DESC to keep the most recently updated record +INSERT INTO nexent.ag_outer_api_services ( + mcp_service_name, + description, + openapi_json, + server_url, + headers_template, + tenant_id, + is_available, + create_time, + update_time, + created_by, + updated_by, + delete_flag +) +SELECT DISTINCT ON (t.tenant_id, t.mcp_service_name) + t.mcp_service_name, + t.description, + t.openapi_json, + t.server_url, + t.headers_template, + t.tenant_id, + COALESCE(t.is_available, TRUE) as is_available, + t.create_time, + t.update_time, + t.created_by, + t.updated_by, + t.delete_flag +FROM nexent.ag_outer_api_tools t +WHERE t.delete_flag != 'Y' +ORDER BY t.tenant_id, t.mcp_service_name, t.update_time DESC +ON CONFLICT DO NOTHING; + +-- Step 3: Verify migration +SELECT 'Migrated services count: ' || COUNT(*) FROM nexent.ag_outer_api_services; + +-- Step 4: Drop old table after successful migration +DROP TABLE IF EXISTS nexent.ag_outer_api_tools; + +-- Step 5: Drop the old sequence (no longer needed) +DROP SEQUENCE IF EXISTS nexent.ag_outer_api_tools_id_seq; + +-- ============================================================================= +-- Add Foreign Key Constraint to ag_a2a_message_t +-- ============================================================================= +-- Version: v2.0.2 +-- Date: 2026-04-20 +-- Description: Add foreign key constraint on task_id referencing ag_a2a_task_t(id) +-- Target Table: nexent.ag_a2a_message_t +-- ============================================================================= + +-- Add foreign key constraint: task_id references ag_a2a_task_t(id) with CASCADE delete +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'ag_a2a_message_t_task_id_fk' + AND conrelid = 'nexent.ag_a2a_message_t'::regclass + ) THEN + ALTER TABLE nexent.ag_a2a_message_t + ADD CONSTRAINT ag_a2a_message_t_task_id_fk + FOREIGN KEY (task_id) + REFERENCES nexent.ag_a2a_task_t(id) ON DELETE CASCADE; + END IF; +END $$; + +-- Add is_a2a column to ag_tenant_agent_version_t for tracking A2A Server agent publish status +-- This field indicates whether this version was published as an A2A Server agent + +ALTER TABLE nexent.ag_tenant_agent_version_t +ADD COLUMN IF NOT EXISTS is_a2a BOOLEAN DEFAULT FALSE; + +COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.is_a2a IS 'Whether this version is published as an A2A Server agent'; + +-- Model Monitoring Record Table +-- Stores per-request LLM performance metrics for the monitoring feature. +-- Run this script against the 'nexent' schema in PostgreSQL. + +CREATE TABLE IF NOT EXISTS nexent.model_monitoring_record_t ( + monitoring_id SERIAL PRIMARY KEY, + model_id INT4, + model_name VARCHAR(100) NOT NULL, + model_type VARCHAR(20) DEFAULT 'llm', + agent_id INT4, + agent_name VARCHAR(100), + conversation_id INT4, + tenant_id VARCHAR(100) NOT NULL, + user_id VARCHAR(100), + display_name VARCHAR(100), + request_duration_ms INT4, + ttft_ms INT4, + input_tokens INT4, + output_tokens INT4, + total_tokens INT4, + generation_rate FLOAT, + is_streaming BOOLEAN DEFAULT FALSE, + is_success BOOLEAN DEFAULT TRUE, + is_error BOOLEAN DEFAULT FALSE, + error_type VARCHAR(50), + error_message TEXT, + retry_count INT4 DEFAULT 0, + operation VARCHAR(50), + create_time TIMESTAMP DEFAULT NOW(), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +-- Single-column indexes for common query patterns +CREATE INDEX IF NOT EXISTS ix_monitoring_model_id ON nexent.model_monitoring_record_t (model_id); +CREATE INDEX IF NOT EXISTS ix_monitoring_tenant_id ON nexent.model_monitoring_record_t (tenant_id); +CREATE INDEX IF NOT EXISTS ix_monitoring_agent_id ON nexent.model_monitoring_record_t (agent_id); +CREATE INDEX IF NOT EXISTS ix_monitoring_create_time ON nexent.model_monitoring_record_t (create_time); +CREATE INDEX IF NOT EXISTS ix_monitoring_is_error ON nexent.model_monitoring_record_t (is_error); +CREATE INDEX IF NOT EXISTS ix_monitoring_model_type ON nexent.model_monitoring_record_t (model_type); + +-- Composite index for time-range queries per model +CREATE INDEX IF NOT EXISTS ix_monitoring_model_time ON nexent.model_monitoring_record_t (model_id, create_time); + +-- Create user OAuth account table for third-party login (GitHub, WeChat, etc.) +CREATE TABLE IF NOT EXISTS nexent.user_oauth_account_t ( + oauth_account_id SERIAL PRIMARY KEY, + user_id VARCHAR(100) NOT NULL, + provider VARCHAR(30) NOT NULL, + provider_user_id VARCHAR(200) NOT NULL, + provider_email VARCHAR(255), + provider_username VARCHAR(200), + tenant_id VARCHAR(100), + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag CHAR(1) DEFAULT 'N', + CONSTRAINT uq_oauth_provider_user UNIQUE (provider, provider_user_id) +); + +ALTER TABLE nexent.user_oauth_account_t OWNER TO "root"; + +-- Create a function to update the update_time column +CREATE OR REPLACE FUNCTION update_user_oauth_account_t_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Create a trigger to call the function before each update +DROP TRIGGER IF EXISTS update_user_oauth_account_t_update_time_trigger ON nexent.user_oauth_account_t; +CREATE TRIGGER update_user_oauth_account_t_update_time_trigger +BEFORE UPDATE ON nexent.user_oauth_account_t +FOR EACH ROW +EXECUTE FUNCTION update_user_oauth_account_t_update_time(); + +-- Add comments +COMMENT ON TABLE nexent.user_oauth_account_t IS 'User OAuth account table - third-party login bindings'; +COMMENT ON COLUMN nexent.user_oauth_account_t.oauth_account_id IS 'OAuth account ID, primary key'; +COMMENT ON COLUMN nexent.user_oauth_account_t.user_id IS 'Nexent user ID (Supabase UUID)'; +COMMENT ON COLUMN nexent.user_oauth_account_t.provider IS 'OAuth provider name: github, wechat, gde, link_app'; +COMMENT ON COLUMN nexent.user_oauth_account_t.provider_user_id IS 'User ID from the OAuth provider'; +COMMENT ON COLUMN nexent.user_oauth_account_t.provider_email IS 'Email from the OAuth provider'; +COMMENT ON COLUMN nexent.user_oauth_account_t.provider_username IS 'Display name from the OAuth provider'; +COMMENT ON COLUMN nexent.user_oauth_account_t.tenant_id IS 'Tenant ID at time of linking'; +COMMENT ON COLUMN nexent.user_oauth_account_t.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.user_oauth_account_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.user_oauth_account_t.created_by IS 'Creator'; +COMMENT ON COLUMN nexent.user_oauth_account_t.updated_by IS 'Updater'; +COMMENT ON COLUMN nexent.user_oauth_account_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; + +-- Create index for user_id queries +CREATE INDEX IF NOT EXISTS idx_user_oauth_account_t_user_id +ON nexent.user_oauth_account_t (user_id); + +-- Migration: Add enable_context_manager column to ag_tenant_agent_t table +-- Date: 2025-04-27 +-- Description: Add enable_context_manager field to control context management (compression) per agent + +-- Add enable_context_manager column to ag_tenant_agent_t table +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS enable_context_manager BOOLEAN DEFAULT FALSE; + +-- Add comment to the column +COMMENT ON COLUMN nexent.ag_tenant_agent_t.enable_context_manager IS 'Whether to enable context management (compression) for this agent'; + +ALTER TABLE nexent.ag_a2a_external_agent_t +ADD COLUMN IF NOT EXISTS base_url VARCHAR(512); + +COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.base_url IS 'Base URL for health checks (service root address)'; + +ALTER TABLE nexent.ag_a2a_message_t + DROP CONSTRAINT IF EXISTS ag_a2a_message_t_task_id_fk; + +ALTER TABLE nexent.ag_a2a_external_agent_relation_t + DROP CONSTRAINT IF EXISTS fk_external_agent; + +ALTER TABLE nexent.ag_a2a_artifact_t + DROP CONSTRAINT IF EXISTS fk_artifact_task; + +-- Migration: Add auto-summary fields to knowledge_record_t table +-- Date: 2026-05-11 +-- Description: Add summary_frequency, last_summary_time, and last_doc_update_time fields for auto-summary feature +-- This SQL consolidates fields added in multiple commits for clean upgrade path + +-- Add summary_frequency column (auto-summary frequency configuration) +ALTER TABLE nexent.knowledge_record_t +ADD COLUMN IF NOT EXISTS summary_frequency VARCHAR(10); + +-- Add last_summary_time column (timestamp of last summary generation) +ALTER TABLE nexent.knowledge_record_t +ADD COLUMN IF NOT EXISTS last_summary_time TIMESTAMP; + +-- Add last_doc_update_time column (timestamp of last document add/delete operation) +ALTER TABLE nexent.knowledge_record_t +ADD COLUMN IF NOT EXISTS last_doc_update_time TIMESTAMP; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.knowledge_record_t.summary_frequency IS 'Auto-summary frequency: 1h, 3h, 6h, 1d, 1w, or NULL (disabled)'; +COMMENT ON COLUMN nexent.knowledge_record_t.last_summary_time IS 'Timestamp of last summary generation'; +COMMENT ON COLUMN nexent.knowledge_record_t.last_doc_update_time IS 'Timestamp of last document add/delete operation, used for auto-summary optimization to skip unnecessary summary regeneration'; diff --git a/docker/sql/v2.1.0_0503_add_prompt_template_t.sql b/deploy/sql/migrations/v2.1_merged_migrations.sql similarity index 83% rename from docker/sql/v2.1.0_0503_add_prompt_template_t.sql rename to deploy/sql/migrations/v2.1_merged_migrations.sql index 3db9a9701..c32e9774c 100644 --- a/docker/sql/v2.1.0_0503_add_prompt_template_t.sql +++ b/deploy/sql/migrations/v2.1_merged_migrations.sql @@ -1,3 +1,6 @@ +-- Nexent merged SQL migrations: v2.1 +-- This file is generated from historical migration files. + -- Migration: Add prompt template table and agent prompt template fields -- Date: 2026-05-03 -- Description: Add user-scoped prompt template storage and bind selected prompt template to agents @@ -113,3 +116,23 @@ ON CONFLICT (template_id) DO UPDATE SET template_content_en = EXCLUDED.template_content_en, updated_by = EXCLUDED.updated_by, delete_flag = 'N'; + +-- Add embedding_model_id column to knowledge_record_t table +-- This field stores the ID of the embedding model used by the knowledge base + +-- Add embedding_model_id column +ALTER TABLE "knowledge_record_t" +ADD COLUMN IF NOT EXISTS "embedding_model_id" INTEGER; + +-- Add column comment +COMMENT ON COLUMN "knowledge_record_t"."embedding_model_id" IS 'Embedding model ID, foreign key reference to model_record_t.model_id'; + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS model_appid VARCHAR(100) DEFAULT ''; + + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS access_token VARCHAR(100) DEFAULT ''; + +COMMENT ON COLUMN nexent.model_record_t.model_appid IS 'Application ID for model authentication.'; +COMMENT ON COLUMN nexent.model_record_t.access_token IS 'Access token for model authentication.'; diff --git a/deploy/sql/migrations/v2.2.0_0615_context_management_capacity_schema.sql b/deploy/sql/migrations/v2.2.0_0615_context_management_capacity_schema.sql new file mode 100644 index 000000000..cc4194d96 --- /dev/null +++ b/deploy/sql/migrations/v2.2.0_0615_context_management_capacity_schema.sql @@ -0,0 +1,144 @@ +-- Migration kind: REQUIRED_SCHEMA +-- Required for: all upgraded deployments before running W1/W2 context-management code. +-- Reason: new code reads/writes these model capacity, monitoring snapshot, and agent override columns. + +-- ============================================================ +-- W1: Add explicit model token-capacity fields to model_record_t +-- ============================================================ +-- All columns are nullable and additive; legacy max_tokens stays as a deprecated +-- output-cap alias until consumers migrate. + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS context_window_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS max_input_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS max_output_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS default_output_reserve_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS tokenizer_family VARCHAR(100) DEFAULT NULL; + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS capacity_source VARCHAR(100) DEFAULT NULL; + +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS capability_profile_version VARCHAR(100) DEFAULT NULL; + +COMMENT ON COLUMN nexent.model_record_t.context_window_tokens IS 'Total combined input/output context window in tokens, when the provider uses a combined window. Nullable.'; +COMMENT ON COLUMN nexent.model_record_t.max_input_tokens IS 'Provider hard input-token limit when distinct from the combined window. Nullable.'; +COMMENT ON COLUMN nexent.model_record_t.max_output_tokens IS 'Provider-supported or operator-configured completion-output cap. Replaces the ambiguous LLM meaning of max_tokens. Nullable.'; +COMMENT ON COLUMN nexent.model_record_t.default_output_reserve_tokens IS 'Default output allowance reserved per request before constructing input context. Nullable.'; +COMMENT ON COLUMN nexent.model_record_t.tokenizer_family IS 'Token-counting strategy or provider/model tokenizer identifier mapped via tokenizer_registry. Nullable.'; +COMMENT ON COLUMN nexent.model_record_t.capacity_source IS 'Source of the persisted capacity value. Optional values: operator, profile, provider_candidate, legacy, unknown.'; +COMMENT ON COLUMN nexent.model_record_t.capability_profile_version IS 'Version of the approved provider/model capability profile used by the request, e.g. openai/gpt-4o@1.'; + +-- ============================================================ +-- W1: Persist resolved model capacity snapshot fields on monitoring records +-- ============================================================ + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS context_window_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS default_output_reserve_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS capability_profile_version VARCHAR(100) DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS capacity_source VARCHAR(100) DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS requested_output_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS provider_input_limit_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS tokenizer_family VARCHAR(100) DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS counting_mode VARCHAR(20) DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS unknown_capabilities JSONB DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS capacity_fingerprint VARCHAR(64) DEFAULT NULL; + +COMMENT ON COLUMN nexent.model_monitoring_record_t.context_window_tokens IS 'Resolved total combined model context window for this request'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.default_output_reserve_tokens IS 'Default output allowance reserved before input context construction'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.capability_profile_version IS 'Version of the resolved capacity profile for this request'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.capacity_source IS 'Dominant source of resolved capacity fields for this request'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.requested_output_tokens IS 'Output tokens requested or reserved during capacity resolution'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.provider_input_limit_tokens IS 'Resolved provider input-token limit used by context management'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.tokenizer_family IS 'Tokenizer family used for request token counting'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.counting_mode IS 'Token counting mode for the request: exact or estimated'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.unknown_capabilities IS 'Structured list of capacity capabilities unknown at resolution time'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.capacity_fingerprint IS 'Fingerprint of the resolved model capacity snapshot'; + +-- ============================================================ +-- W2: Add per-agent requested_output_tokens override +-- ============================================================ + +ALTER TABLE nexent.ag_tenant_agent_t + ADD COLUMN IF NOT EXISTS requested_output_tokens INTEGER NULL; + +COMMENT ON COLUMN nexent.ag_tenant_agent_t.requested_output_tokens IS + 'Per-agent override for W2 requested_output_tokens. NULL means inherit ' + 'the resolved model-level default. Must satisfy 0 < value <= ' + 'max_output_tokens from the resolved W1 capacity at save time.'; + +-- ============================================================ +-- W2: Add safe input budget snapshot fields to model monitoring records +-- ============================================================ + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_fingerprint VARCHAR(64) DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_w1_fingerprint VARCHAR(64) DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_requested_output_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_output_reserve_source VARCHAR(32) DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_provider_input_limit_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_uncertainty_reserve_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_uncertainty_reserve_basis VARCHAR(64) DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_soft_limit_ratio FLOAT DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_soft_input_budget_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_hard_input_budget_tokens INTEGER DEFAULT NULL; + +ALTER TABLE nexent.model_monitoring_record_t +ADD COLUMN IF NOT EXISTS budget_warnings JSONB DEFAULT NULL; + +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_fingerprint IS 'Fingerprint of the resolved W2 safe input budget snapshot'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_w1_fingerprint IS 'W1 capacity fingerprint consumed by the W2 budget snapshot'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_requested_output_tokens IS 'W2 trusted requested output tokens used at dispatch'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_output_reserve_source IS 'Source of the W2 requested output token reserve'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_provider_input_limit_tokens IS 'Provider input limit after applying the W2 output reserve'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_uncertainty_reserve_tokens IS 'Additional W2 uncertainty reserve deducted from input budget'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_uncertainty_reserve_basis IS 'Basis used for the W2 uncertainty reserve'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_soft_limit_ratio IS 'W2 soft input budget ratio'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_soft_input_budget_tokens IS 'W2 soft input budget where proactive compression begins'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_hard_input_budget_tokens IS 'W2 hard input budget consumed by W3 final fit'; +COMMENT ON COLUMN nexent.model_monitoring_record_t.budget_warnings IS 'Structured W2 budget warnings active for this request'; diff --git a/deploy/sql/migrations/v2.2.0_0617_context_management_capacity_data_fix.sql b/deploy/sql/migrations/v2.2.0_0617_context_management_capacity_data_fix.sql new file mode 100644 index 000000000..371a2fed3 --- /dev/null +++ b/deploy/sql/migrations/v2.2.0_0617_context_management_capacity_data_fix.sql @@ -0,0 +1,205 @@ +-- Migration kind: RECOMMENDED_DATA_FIX +-- Required for: upgraded deployments with existing model_record_t rows. +-- Safe to skip when: fresh deployment, or operators will manually fill capacity fields. +-- Reason: improves legacy model capacity completeness and reconciles the temporary max_tokens alias. +-- +-- ------------------------------------------------------------ +-- Pre-run self-check (recommended before applying) +-- ------------------------------------------------------------ +-- The reconcile block at the bottom of this file rewrites `max_tokens` to +-- match the freshly backfilled `max_output_tokens`. If an operator +-- previously tightened `max_tokens` below the catalog value on a row this +-- migration touches (cost control, prompt-budget caps, etc.), that tighter +-- value will be overwritten with the catalog value. +-- +-- Run this query first to surface any such rows: +-- +-- SELECT model_id, model_name, model_factory, max_tokens, max_output_tokens +-- FROM nexent.model_record_t +-- WHERE delete_flag = 'N' +-- AND max_tokens IS NOT NULL +-- AND ( +-- (LOWER(model_factory)='openai' AND model_name IN ('gpt-4o','gpt-4.1')) +-- OR (LOWER(model_factory)='dashscope' AND model_name IN ('qwen-plus','qwen-turbo','qwen3.7-max','glm-5.1')) +-- OR (LOWER(model_factory)='silicon' AND model_name IN ('Qwen/Qwen3.6-27B','Pro/moonshotai/Kimi-K2.6')) +-- OR (LOWER(model_factory)='deepseek' AND model_name IN ('deepseek-v4-flash','deepseek-v4-pro')) +-- ); +-- +-- If the result is empty: safe to apply the whole file. +-- If the result has rows the operator deliberately tightened: run only the +-- first `DO $$` block (catalog backfill) and skip the second (reconcile), +-- or back up the affected rows before applying. + +-- ============================================================ +-- Backfill capacity columns on legacy model_record_t rows +-- ============================================================ +-- Matches (model_factory, model_name) against W1 day-one catalog entries. +-- Idempotent: only writes when context_window_tokens IS NULL, so re-running on +-- already-backfilled rows is a no-op. +-- +-- Catalog source of truth: backend/consts/capability_profiles.py (W1 ADR +-- Decision 1). If the catalog is bumped, mirror the change here in a new +-- migration; do not edit this file in place after it has been released. +-- +-- Coverage caveat: rows whose model_factory does not match a catalog provider +-- key (commonly the manual-add default 'OpenAI-API-Compatible' per CM-031) +-- will not be backfilled by this migration. Operators must either update +-- model_factory directly, re-save the model through the W1-aware UI, or wait +-- for W17. Startup logs surface the residual count. + +DO $$ +DECLARE + v_updated INTEGER := 0; + v_total INTEGER := 0; +BEGIN + -- openai/gpt-4o + UPDATE nexent.model_record_t + SET context_window_tokens = 128000, + max_output_tokens = 16384, + default_output_reserve_tokens = 4096 + WHERE LOWER(model_factory) = 'openai' + AND model_name = 'gpt-4o' + AND delete_flag = 'N' + AND context_window_tokens IS NULL; + GET DIAGNOSTICS v_updated = ROW_COUNT; + v_total := v_total + v_updated; + + -- openai/gpt-4.1 + UPDATE nexent.model_record_t + SET context_window_tokens = 1000000, + max_output_tokens = 32768, + default_output_reserve_tokens = 8192 + WHERE LOWER(model_factory) = 'openai' + AND model_name = 'gpt-4.1' + AND delete_flag = 'N' + AND context_window_tokens IS NULL; + GET DIAGNOSTICS v_updated = ROW_COUNT; + v_total := v_total + v_updated; + + -- dashscope/qwen-plus + UPDATE nexent.model_record_t + SET context_window_tokens = 131072, + max_output_tokens = 16384, + default_output_reserve_tokens = 4096 + WHERE LOWER(model_factory) = 'dashscope' + AND model_name = 'qwen-plus' + AND delete_flag = 'N' + AND context_window_tokens IS NULL; + GET DIAGNOSTICS v_updated = ROW_COUNT; + v_total := v_total + v_updated; + + -- dashscope/qwen-turbo + UPDATE nexent.model_record_t + SET context_window_tokens = 1000000, + max_output_tokens = 16384, + default_output_reserve_tokens = 4096 + WHERE LOWER(model_factory) = 'dashscope' + AND model_name = 'qwen-turbo' + AND delete_flag = 'N' + AND context_window_tokens IS NULL; + GET DIAGNOSTICS v_updated = ROW_COUNT; + v_total := v_total + v_updated; + + -- dashscope/qwen3.7-max + UPDATE nexent.model_record_t + SET context_window_tokens = 1000000, + max_output_tokens = 65536, + default_output_reserve_tokens = 8192 + WHERE LOWER(model_factory) = 'dashscope' + AND model_name = 'qwen3.7-max' + AND delete_flag = 'N' + AND context_window_tokens IS NULL; + GET DIAGNOSTICS v_updated = ROW_COUNT; + v_total := v_total + v_updated; + + -- dashscope/glm-5.1 + UPDATE nexent.model_record_t + SET context_window_tokens = 200000, + max_output_tokens = 131072, + default_output_reserve_tokens = 8192 + WHERE LOWER(model_factory) = 'dashscope' + AND model_name = 'glm-5.1' + AND delete_flag = 'N' + AND context_window_tokens IS NULL; + GET DIAGNOSTICS v_updated = ROW_COUNT; + v_total := v_total + v_updated; + + -- silicon/Qwen/Qwen3.6-27B + UPDATE nexent.model_record_t + SET context_window_tokens = 262144, + max_output_tokens = 65536, + default_output_reserve_tokens = 8192 + WHERE LOWER(model_factory) = 'silicon' + AND model_name = 'Qwen/Qwen3.6-27B' + AND delete_flag = 'N' + AND context_window_tokens IS NULL; + GET DIAGNOSTICS v_updated = ROW_COUNT; + v_total := v_total + v_updated; + + -- silicon/Pro/moonshotai/Kimi-K2.6 + UPDATE nexent.model_record_t + SET context_window_tokens = 262144, + max_output_tokens = 131072, + default_output_reserve_tokens = 8192 + WHERE LOWER(model_factory) = 'silicon' + AND model_name = 'Pro/moonshotai/Kimi-K2.6' + AND delete_flag = 'N' + AND context_window_tokens IS NULL; + GET DIAGNOSTICS v_updated = ROW_COUNT; + v_total := v_total + v_updated; + + -- deepseek/deepseek-v4-flash + -- (deepseek-chat / deepseek-reasoner intentionally omitted: they alias to + -- v4-flash and are scheduled for deprecation at 2026-07-24, and pre-W1 + -- deployments may have legacy max_tokens values for those names that + -- this backfill should not clobber.) + UPDATE nexent.model_record_t + SET context_window_tokens = 1000000, + max_output_tokens = 384000, + default_output_reserve_tokens = 8192 + WHERE LOWER(model_factory) = 'deepseek' + AND model_name = 'deepseek-v4-flash' + AND delete_flag = 'N' + AND context_window_tokens IS NULL; + GET DIAGNOSTICS v_updated = ROW_COUNT; + v_total := v_total + v_updated; + + -- deepseek/deepseek-v4-pro + UPDATE nexent.model_record_t + SET context_window_tokens = 1000000, + max_output_tokens = 384000, + default_output_reserve_tokens = 8192 + WHERE LOWER(model_factory) = 'deepseek' + AND model_name = 'deepseek-v4-pro' + AND delete_flag = 'N' + AND context_window_tokens IS NULL; + GET DIAGNOSTICS v_updated = ROW_COUNT; + v_total := v_total + v_updated; + + RAISE NOTICE 'W2 catalog backfill: % row(s) updated', v_total; +END $$; + +-- ============================================================ +-- Reconcile the legacy max_tokens column with max_output_tokens +-- ============================================================ +-- Runs after the catalog backfill above because the backfill writes +-- max_output_tokens. Scope and safety: +-- * Only touches rows where max_output_tokens IS NOT NULL. +-- * Skips embedding rows because they reuse max_tokens as the vector dimension. +-- * Only updates rows where the two columns actually disagree. +-- * delete_flag = 'N' so soft-deleted rows are left alone. + +DO $$ +DECLARE + v_updated INTEGER := 0; +BEGIN + UPDATE nexent.model_record_t + SET max_tokens = max_output_tokens + WHERE delete_flag = 'N' + AND max_output_tokens IS NOT NULL + AND COALESCE(max_tokens, -1) <> max_output_tokens + AND COALESCE(model_type, '') NOT IN ('embedding', 'multi_embedding'); + + GET DIAGNOSTICS v_updated = ROW_COUNT; + RAISE NOTICE 'max_tokens alias reconcile: % row(s) updated', v_updated; +END $$; diff --git a/deploy/sql/migrations/v2.2.2_0622_update_left_nav_menu.sql b/deploy/sql/migrations/v2.2.2_0622_update_left_nav_menu.sql new file mode 100644 index 000000000..8dcba06ba --- /dev/null +++ b/deploy/sql/migrations/v2.2.2_0622_update_left_nav_menu.sql @@ -0,0 +1,105 @@ +-- ============================================================ +-- Menu Structure Migration V2 +-- Migration Date: 2026-06-22 +-- ============================================================ + +-- Step 1: Clear all existing LEFT_NAV_MENU permissions +BEGIN; + +DELETE FROM nexent.role_permission_t +WHERE permission_category = 'VISIBILITY' AND permission_type = 'LEFT_NAV_MENU'; + +ALTER TABLE nexent.role_permission_t +ADD COLUMN IF NOT EXISTS parent_key VARCHAR(50); +-- ============================================================ +-- New Menu Structure: +-- ROOT: /, /chat, /agent-dev, /resource-space, /resource-manage, /owner-manage, /users +-- AGENT-DEV: /models, /knowledges, /agents, /memory +-- RESOURCE-SPACE: /agent-space, /mcp-space, /skill-space +-- ============================================================ +-- ID Format: xx +-- SU=10xx, ADMIN=11xx, DEV=12xx, USER=13xx, SPEED=14xx, ASSET_OWNER=15xx +-- parent_key: NULL for first-level, parent route for second-level +-- ============================================================ + +-- SU Menus (root level) +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES +(1001, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(1002, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-manage'); + +-- ADMIN Menus (root level) +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES +(1101, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(1102, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(1103, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/agent-dev'), +(1104, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-space'), +(1105, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-manage'), +(1106, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'); +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) VALUES +(1107, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/models', '/agent-dev'), +(1108, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges', '/agent-dev'), +(1109, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents', '/agent-dev'), +(1110, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory', '/agent-dev'); +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) VALUES +(1111, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/agent-space', '/resource-space'), +(1112, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-space', '/resource-space'), +(1113, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/skill-space', '/resource-space'); + +-- DEV Menus (NO /resource-manage, root level) +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES +(1201, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(1202, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(1203, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/agent-dev'), +(1204, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-space'), +(1205, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'); +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) VALUES +(1206, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/models', '/agent-dev'), +(1207, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges', '/agent-dev'), +(1208, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents', '/agent-dev'), +(1209, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory', '/agent-dev'); +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) VALUES +(1210, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/agent-space', '/resource-space'), +(1211, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-space', '/resource-space'), +(1212, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/skill-space', '/resource-space'); + +-- USER Menus (Minimal, all root level) +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES +(1301, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(1302, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(1303, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), +(1304, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'); + +-- SPEED Menus (root level) +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES +(1401, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(1402, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(1403, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/agent-dev'), +(1404, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-space'), +(1405, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-manage'); +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) VALUES +(1406, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/models', '/agent-dev'), +(1407, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges', '/agent-dev'), +(1408, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents', '/agent-dev'), +(1409, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory', '/agent-dev'); +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) VALUES +(1410, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/agent-space', '/resource-space'), +(1411, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-space', '/resource-space'), +(1412, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/skill-space', '/resource-space'); + +-- ASSET_OWNER Menus (root level) +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES +(1501, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), +(1502, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), +(1503, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/agent-dev'), +(1504, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-space'), +(1505, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/owner-manage'); +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) VALUES +(1506, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/models', '/agent-dev'), +(1507, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges', '/agent-dev'), +(1508, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents', '/agent-dev'); +INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) VALUES +(1509, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/agent-space', '/resource-space'), +(1510, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-space', '/resource-space'), +(1511, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/skill-space', '/resource-space'); + +COMMIT; diff --git a/deploy/sql/migrations/v2.2_merged_migrations.sql b/deploy/sql/migrations/v2.2_merged_migrations.sql new file mode 100644 index 000000000..2c134da51 --- /dev/null +++ b/deploy/sql/migrations/v2.2_merged_migrations.sql @@ -0,0 +1,475 @@ +-- Nexent merged SQL migrations: v2.2 +-- This file is generated from historical migration files. + +-- Rename params -> config_values, add config_schemas to ag_skill_info_t +-- Add tenant_id column for multi-tenancy support +ALTER TABLE nexent.ag_skill_info_t ADD COLUMN IF NOT EXISTS tenant_id VARCHAR(100); + +-- Add config_values and config_schemas to ag_skill_info_t +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'nexent' + AND table_name = 'ag_skill_info_t' + AND column_name = 'params' + ) AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'nexent' + AND table_name = 'ag_skill_info_t' + AND column_name = 'config_values' + ) THEN + ALTER TABLE nexent.ag_skill_info_t RENAME COLUMN params TO config_values; + ELSIF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'nexent' + AND table_name = 'ag_skill_info_t' + AND column_name = 'params' + ) AND EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'nexent' + AND table_name = 'ag_skill_info_t' + AND column_name = 'config_values' + ) THEN + UPDATE nexent.ag_skill_info_t + SET config_values = params + WHERE config_values IS NULL + AND params IS NOT NULL; + END IF; +END $$; +ALTER TABLE nexent.ag_skill_info_t ADD COLUMN IF NOT EXISTS config_values JSON; +ALTER TABLE nexent.ag_skill_info_t ADD COLUMN IF NOT EXISTS config_schemas JSON; + +-- Comments for ag_skill_info_t columns +COMMENT ON COLUMN nexent.ag_skill_info_t.tenant_id IS 'Tenant ID for multi-tenancy. NULL for pre-existing skills.'; +COMMENT ON COLUMN nexent.ag_skill_info_t.config_values IS 'Runtime parameter values from config/config.yaml'; +COMMENT ON COLUMN nexent.ag_skill_info_t.config_schemas IS 'Parameter metadata list from config/schema.yaml'; + +-- Add config_values and config_schemas to ag_skill_instance_t +ALTER TABLE nexent.ag_skill_instance_t ADD COLUMN IF NOT EXISTS config_values JSON; +ALTER TABLE nexent.ag_skill_instance_t ADD COLUMN IF NOT EXISTS config_schemas JSON; + +-- Comments for ag_skill_instance_t columns +COMMENT ON COLUMN nexent.ag_skill_instance_t.config_values IS 'Per-agent runtime parameter values from config/config.yaml'; +COMMENT ON COLUMN nexent.ag_skill_instance_t.config_schemas IS 'Per-agent parameter schema overrides from config/schema.yaml'; + +-- Add concurrency_limit column to model_record_t table +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS concurrency_limit INTEGER DEFAULT NULL; + +-- Add comment to the column +COMMENT ON COLUMN nexent.model_record_t.concurrency_limit IS 'Maximum concurrent requests for this model. Default is NULL (unlimited).'; + +-- Add timeout_seconds column to model_record_t table +ALTER TABLE nexent.model_record_t +ADD COLUMN IF NOT EXISTS timeout_seconds INTEGER DEFAULT 120; + +-- Add comment to the column +COMMENT ON COLUMN nexent.model_record_t.timeout_seconds IS 'Request timeout in seconds for this model. Default is 120 seconds.'; + +-- Migration: Add mcp_community_record_t table +-- Date: 2026-03-26 +-- Description: Community MCP market table aligned with public-shareable fields from mcp_record_t. + +SET search_path TO nexent; + +BEGIN; + +CREATE TABLE IF NOT EXISTS nexent.mcp_community_record_t ( + community_id SERIAL PRIMARY KEY NOT NULL, + tenant_id VARCHAR(100), + user_id VARCHAR(100), + mcp_name VARCHAR(100) NOT NULL, + mcp_server VARCHAR(500) NOT NULL, + source VARCHAR(30) DEFAULT 'community', + version VARCHAR(50), + registry_json JSONB, + transport_type VARCHAR(30), + config_json JSON, + tags TEXT[], + description TEXT, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE nexent.mcp_community_record_t OWNER TO root; + +COMMENT ON TABLE nexent.mcp_community_record_t IS 'Community MCP market records, publishable from tenant MCP services'; +COMMENT ON COLUMN nexent.mcp_community_record_t.community_id IS 'Community record ID, unique primary key'; +COMMENT ON COLUMN nexent.mcp_community_record_t.tenant_id IS 'Publisher tenant ID'; +COMMENT ON COLUMN nexent.mcp_community_record_t.user_id IS 'Publisher user ID'; +COMMENT ON COLUMN nexent.mcp_community_record_t.mcp_name IS 'MCP name'; +COMMENT ON COLUMN nexent.mcp_community_record_t.mcp_server IS 'MCP server URL'; +COMMENT ON COLUMN nexent.mcp_community_record_t.source IS 'Source type, fixed to community for this table'; +COMMENT ON COLUMN nexent.mcp_community_record_t.version IS 'MCP version'; +COMMENT ON COLUMN nexent.mcp_community_record_t.registry_json IS 'Full MCP server metadata JSON for discovery and quick import'; +COMMENT ON COLUMN nexent.mcp_community_record_t.transport_type IS 'Transport type: url/container'; +COMMENT ON COLUMN nexent.mcp_community_record_t.config_json IS 'Public-shareable MCP configuration JSON'; +COMMENT ON COLUMN nexent.mcp_community_record_t.tags IS 'Tags'; +COMMENT ON COLUMN nexent.mcp_community_record_t.description IS 'Description'; +COMMENT ON COLUMN nexent.mcp_community_record_t.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.mcp_community_record_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.mcp_community_record_t.created_by IS 'Creator ID'; +COMMENT ON COLUMN nexent.mcp_community_record_t.updated_by IS 'Updater ID'; +COMMENT ON COLUMN nexent.mcp_community_record_t.delete_flag IS 'Soft delete flag: Y/N'; + +CREATE INDEX IF NOT EXISTS idx_mcp_community_tenant_delete + ON nexent.mcp_community_record_t (tenant_id, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_mcp_community_name_delete + ON nexent.mcp_community_record_t (mcp_name, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_mcp_community_transport_delete + ON nexent.mcp_community_record_t (transport_type, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_mcp_community_user_delete + ON nexent.mcp_community_record_t (user_id, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_mcp_community_tags_gin + ON nexent.mcp_community_record_t USING GIN (tags); + +CREATE OR REPLACE FUNCTION update_mcp_community_record_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +COMMENT ON FUNCTION update_mcp_community_record_update_time() IS 'Auto-update update_time for mcp_community_record_t'; + +DROP TRIGGER IF EXISTS update_mcp_community_record_update_time_trigger ON nexent.mcp_community_record_t; +CREATE TRIGGER update_mcp_community_record_update_time_trigger +BEFORE UPDATE ON nexent.mcp_community_record_t +FOR EACH ROW +EXECUTE FUNCTION update_mcp_community_record_update_time(); + +COMMENT ON TRIGGER update_mcp_community_record_update_time_trigger ON nexent.mcp_community_record_t IS 'Trigger to maintain update_time'; + +COMMIT; + +-- Migration: Extend mcp_record_t for MCP tools (direct schema) +-- Date: 2026-03-18 +-- Description: One-step schema extension for mcp_record_t. No table merge, no data migration. + +SET search_path TO nexent; + +BEGIN; + +-- 1) Extend mcp_record_t with final column names (idempotent) +ALTER TABLE IF EXISTS nexent.mcp_record_t + ADD COLUMN IF NOT EXISTS source VARCHAR(30), + ADD COLUMN IF NOT EXISTS registry_json JSONB, + ADD COLUMN IF NOT EXISTS config_json JSON, + ADD COLUMN IF NOT EXISTS enabled BOOLEAN DEFAULT TRUE, + ADD COLUMN IF NOT EXISTS tags TEXT[], + ADD COLUMN IF NOT EXISTS description TEXT, + ADD COLUMN IF NOT EXISTS container_port INTEGER; + +-- 2) Add comments for new columns +COMMENT ON COLUMN nexent.mcp_record_t.source IS 'Source type: local/mcp_registry/community'; +COMMENT ON COLUMN nexent.mcp_record_t.registry_json IS 'Full MCP registry server.json snapshot'; +COMMENT ON COLUMN nexent.mcp_record_t.config_json IS 'MCP config data'; +COMMENT ON COLUMN nexent.mcp_record_t.enabled IS 'Enabled'; +COMMENT ON COLUMN nexent.mcp_record_t.tags IS 'Tags'; +COMMENT ON COLUMN nexent.mcp_record_t.description IS 'Description'; +COMMENT ON COLUMN nexent.mcp_record_t.container_port IS 'Host port bound for containerized MCP service'; + +-- 3) Add indexes for common management queries +CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tenant_delete + ON nexent.mcp_record_t (tenant_id, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tenant_name + ON nexent.mcp_record_t (tenant_id, mcp_name, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tenant_server + ON nexent.mcp_record_t (tenant_id, mcp_server, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tags_gin + ON nexent.mcp_record_t USING GIN (tags); + +COMMIT; + +CREATE TABLE IF NOT EXISTS nexent.user_cas_session_t ( + cas_session_id SERIAL PRIMARY KEY, + session_id VARCHAR(100) NOT NULL UNIQUE, + user_id VARCHAR(100) NOT NULL, + cas_user_id VARCHAR(200) NOT NULL, + cas_session_index VARCHAR(500), + status VARCHAR(30) NOT NULL DEFAULT 'active', + expires_at TIMESTAMP NOT NULL, + revoked_at TIMESTAMP, + create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +CREATE INDEX IF NOT EXISTS ix_user_cas_session_session_id + ON nexent.user_cas_session_t (session_id); +CREATE INDEX IF NOT EXISTS ix_user_cas_session_user_id + ON nexent.user_cas_session_t (user_id); +CREATE INDEX IF NOT EXISTS ix_user_cas_session_cas_user_id + ON nexent.user_cas_session_t (cas_user_id); + +COMMENT ON TABLE nexent.user_cas_session_t IS 'Server-side session records for CAS SSO login and logout synchronization'; +COMMENT ON COLUMN nexent.user_cas_session_t.session_id IS 'JWT sid claim for revocation checks'; +COMMENT ON COLUMN nexent.user_cas_session_t.cas_user_id IS 'User identifier returned by CAS'; +COMMENT ON COLUMN nexent.user_cas_session_t.cas_session_index IS 'CAS SessionIndex or service ticket'; + +-- Migration: Add custom_headers column to mcp_record_t +-- Date: 2026-05-26 +-- Description: Add custom_headers field to store custom HTTP headers for MCP server requests + +SET search_path TO nexent; + +BEGIN; + +-- Add custom_headers column if it doesn't exist +ALTER TABLE nexent.mcp_record_t +ADD COLUMN IF NOT EXISTS custom_headers JSON DEFAULT NULL; + +-- Add comment to the column +COMMENT ON COLUMN nexent.mcp_record_t.custom_headers IS 'Custom HTTP headers as JSON object for MCP server requests'; + +COMMIT; + +-- Migration: ASSET_OWNER role permissions and invitation type comment +-- Date: 2026-05-29 +-- Description: Add ASSET_OWNER role permissions, SU asset-owner invite permissions, +-- update invitation code_type comment, and ensure ag_skill_info_t.tenant_id exists +-- Source: commit 15cece97692db2372a978cbdf21b5d5316e79f30 (init.sql) + +SET search_path TO nexent; + +BEGIN; + +COMMENT ON COLUMN nexent.tenant_invitation_code_t.code_type IS + 'Invitation code type: ADMIN_INVITE, DEV_INVITE, USER_INVITE, ASSET_OWNER_INVITE'; + +INSERT INTO nexent.role_permission_t + (role_permission_id, user_role, permission_category, permission_type, permission_subtype) +VALUES + (188, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'CREATE'), + (189, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'READ'), + (190, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'UPDATE'), + (191, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'DELETE'), + (192, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), + (193, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), + (194, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), + (195, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), + (196, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), + (197, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), + (198, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), + (199, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'CREATE'), + (200, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'READ'), + (201, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'UPDATE'), + (202, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'DELETE'), + (203, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'CREATE'), + (204, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'READ'), + (205, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'UPDATE'), + (206, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'DELETE'), + (207, 'ASSET_OWNER', 'RESOURCE', 'KB', 'CREATE'), + (208, 'ASSET_OWNER', 'RESOURCE', 'KB', 'READ'), + (209, 'ASSET_OWNER', 'RESOURCE', 'KB', 'UPDATE'), + (210, 'ASSET_OWNER', 'RESOURCE', 'KB', 'DELETE'), + (211, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'CREATE'), + (212, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'READ'), + (213, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'UPDATE'), + (214, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'DELETE'), + (215, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'CREATE'), + (216, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'READ'), + (217, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'UPDATE'), + (218, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'DELETE'), + (219, 'ASSET_OWNER', 'RESOURCE', 'USER.ROLE', 'READ'), + (220, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), + (221, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/asset-owner-resources') +ON CONFLICT (role_permission_id) DO NOTHING; + +COMMIT; + +-- Migration: Add layered ReAct self-verification config to agents +-- Description: Stores per-agent verification controls for step-level and final-answer validation. + +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS verification_config JSONB; + +COMMENT ON COLUMN nexent.ag_tenant_agent_t.verification_config IS 'Layered ReAct self-verification configuration'; + +-- Migration: Add preserve_source_file to knowledge_record_t table +-- Date: 2026-06-01 +-- Description: Whether to preserve uploaded source documents after vectorization (default: true) + +ALTER TABLE nexent.knowledge_record_t +ADD COLUMN IF NOT EXISTS preserve_source_file BOOLEAN NOT NULL DEFAULT true; + +COMMENT ON COLUMN nexent.knowledge_record_t.preserve_source_file IS 'Whether to preserve uploaded source documents after vectorization'; + +-- Migration: Add greeting_message and example_questions columns to ag_tenant_agent_t table +-- Date: 2026-06-03 +-- Description: Add greeting message and example questions fields for agent chat initial screen + +-- Add greeting_message column to ag_tenant_agent_t table +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS greeting_message TEXT; + +-- Add example_questions column to ag_tenant_agent_t table +ALTER TABLE nexent.ag_tenant_agent_t +ADD COLUMN IF NOT EXISTS example_questions JSONB; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.ag_tenant_agent_t.greeting_message IS 'Agent greeting message displayed on chat initial screen'; +COMMENT ON COLUMN nexent.ag_tenant_agent_t.example_questions IS 'List of example questions for starting a conversation with this agent'; + +-- Migration: Add ag_agent_repository_t table +-- Date: 2026-06-05 +-- Description: Agent marketplace repository for frozen shareable agent snapshots. + +SET search_path TO nexent; + +BEGIN; + +CREATE SEQUENCE IF NOT EXISTS nexent.ag_agent_repository_t_agent_repository_id_seq; + +CREATE TABLE IF NOT EXISTS nexent.ag_agent_repository_t ( + agent_repository_id BIGINT NOT NULL DEFAULT nextval('nexent.ag_agent_repository_t_agent_repository_id_seq'), + publisher_tenant_id VARCHAR(100) NOT NULL, + publisher_user_id VARCHAR(100) NOT NULL, + agent_id INTEGER NOT NULL, + version_no INTEGER NOT NULL, + name VARCHAR(100) NOT NULL, + display_name VARCHAR(100), + description TEXT, + author VARCHAR(100), + submitted_by VARCHAR(100), + category_id INTEGER, + tags TEXT[], + tool_count INTEGER, + icon VARCHAR(100), + downloads INTEGER DEFAULT 0, + version_name VARCHAR(100), + agent_info_json JSONB NOT NULL, + status VARCHAR(30) DEFAULT 'not_shared', + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N', + CONSTRAINT ag_agent_repository_t_pkey PRIMARY KEY (agent_repository_id) +); + +ALTER SEQUENCE nexent.ag_agent_repository_t_agent_repository_id_seq + OWNED BY nexent.ag_agent_repository_t.agent_repository_id; + +ALTER TABLE nexent.ag_agent_repository_t OWNER TO root; + +-- Upgrade legacy ag_agent_repository_t schema if table already exists +DO $$ BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'nexent' AND table_name = 'ag_agent_repository_t' + AND column_name = 'source_version_no' + ) THEN + ALTER TABLE nexent.ag_agent_repository_t + RENAME COLUMN source_version_no TO version_no; + END IF; +END $$; + +DO $$ BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'nexent' AND table_name = 'ag_agent_repository_t' + AND column_name = 'version_label' + ) THEN + ALTER TABLE nexent.ag_agent_repository_t + RENAME COLUMN version_label TO version_name; + END IF; +END $$; + +ALTER TABLE nexent.ag_agent_repository_t + ADD COLUMN IF NOT EXISTS submitted_by VARCHAR(100), + ADD COLUMN IF NOT EXISTS icon VARCHAR(100), + ADD COLUMN IF NOT EXISTS downloads INTEGER DEFAULT 0; + +DROP INDEX IF EXISTS nexent.uq_agent_repository_tenant_agent_active; + +COMMENT ON TABLE nexent.ag_agent_repository_t IS 'Agent marketplace repository for frozen shareable agent snapshots'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.agent_repository_id IS 'Agent repository listing ID, unique primary key'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.publisher_tenant_id IS 'Publisher tenant ID'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.publisher_user_id IS 'Publisher user ID'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.agent_id IS 'Root agent ID from ag_tenant_agent_t; unique per version_no when active (delete_flag = N)'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.version_no IS 'Published version number frozen at share time'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.name IS 'Root agent programmatic name for display and search'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.display_name IS 'Root agent display name'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.description IS 'Root agent description'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.author IS 'Agent author'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.submitted_by IS 'Submitter email when listing enters pending_review'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.category_id IS 'Optional marketplace category ID'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.tags IS 'Marketplace tags'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.tool_count IS 'Total tool count across all agents in the bundle (display only)'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.version_name IS 'Repository entry version name for display (from ag_tenant_agent_version_t)'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.icon IS 'Marketplace card icon (emoji or URL)'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.downloads IS 'Marketplace download/copy count for card display'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.agent_info_json IS 'Frozen ExportAndImportDataFormat snapshot with optional skills'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.status IS 'Listing status: not_shared (未共享) / pending_review (待审核) / rejected (审核驳回) / shared (已共享)'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.created_by IS 'Creator ID'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.updated_by IS 'Updater ID'; +COMMENT ON COLUMN nexent.ag_agent_repository_t.delete_flag IS 'Soft delete flag: Y/N'; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_agent_repository_agent_version_active + ON nexent.ag_agent_repository_t (agent_id, version_no) + WHERE delete_flag = 'N'; + +CREATE INDEX IF NOT EXISTS idx_agent_repository_publisher_delete + ON nexent.ag_agent_repository_t (publisher_tenant_id, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_agent_repository_status_delete + ON nexent.ag_agent_repository_t (status, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_agent_repository_name_delete + ON nexent.ag_agent_repository_t (name, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_agent_repository_tags_gin + ON nexent.ag_agent_repository_t USING GIN (tags); + +CREATE OR REPLACE FUNCTION update_ag_agent_repository_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +COMMENT ON FUNCTION update_ag_agent_repository_update_time() IS 'Auto-update update_time for ag_agent_repository_t'; + +DROP TRIGGER IF EXISTS update_ag_agent_repository_update_time_trigger ON nexent.ag_agent_repository_t; +CREATE TRIGGER update_ag_agent_repository_update_time_trigger +BEFORE UPDATE ON nexent.ag_agent_repository_t +FOR EACH ROW +EXECUTE FUNCTION update_ag_agent_repository_update_time(); + +COMMENT ON TRIGGER update_ag_agent_repository_update_time_trigger ON nexent.ag_agent_repository_t IS 'Trigger to maintain update_time'; + +COMMIT; + +-- Migration: Add selected_agent_version_no to ag_agent_relation_t +-- Date: 2026-06-09 +-- Description: Pin child agent version on parent-child relations at publish time. + +SET search_path TO nexent; + +BEGIN; + +ALTER TABLE nexent.ag_agent_relation_t + ADD COLUMN IF NOT EXISTS selected_agent_version_no INTEGER; + +COMMENT ON COLUMN nexent.ag_agent_relation_t.selected_agent_version_no IS + 'Pinned version of selected_agent_id. NULL = use child current published version at runtime (legacy/draft).'; + +COMMIT; diff --git a/docker/volumes/db/_supabase.sql b/deploy/sql/supabase/_supabase.sql similarity index 100% rename from docker/volumes/db/_supabase.sql rename to deploy/sql/supabase/_supabase.sql diff --git a/docker/volumes/db/init/data.sql b/deploy/sql/supabase/init/data.sql similarity index 100% rename from docker/volumes/db/init/data.sql rename to deploy/sql/supabase/init/data.sql diff --git a/docker/volumes/db/jwt.sql b/deploy/sql/supabase/jwt.sql similarity index 100% rename from docker/volumes/db/jwt.sql rename to deploy/sql/supabase/jwt.sql diff --git a/docker/volumes/db/logs.sql b/deploy/sql/supabase/logs.sql similarity index 100% rename from docker/volumes/db/logs.sql rename to deploy/sql/supabase/logs.sql diff --git a/docker/volumes/db/pooler.sql b/deploy/sql/supabase/pooler.sql similarity index 100% rename from docker/volumes/db/pooler.sql rename to deploy/sql/supabase/pooler.sql diff --git a/docker/volumes/db/realtime.sql b/deploy/sql/supabase/realtime.sql similarity index 100% rename from docker/volumes/db/realtime.sql rename to deploy/sql/supabase/realtime.sql diff --git a/docker/volumes/db/roles.sql b/deploy/sql/supabase/roles.sql similarity index 100% rename from docker/volumes/db/roles.sql rename to deploy/sql/supabase/roles.sql diff --git a/docker/volumes/db/webhooks.sql b/deploy/sql/supabase/webhooks.sql similarity index 92% rename from docker/volumes/db/webhooks.sql rename to deploy/sql/supabase/webhooks.sql index cf2ee1079..f07f82fa4 100644 --- a/docker/volumes/db/webhooks.sql +++ b/deploy/sql/supabase/webhooks.sql @@ -2,30 +2,31 @@ BEGIN; -- Create pg_net extension CREATE EXTENSION IF NOT EXISTS pg_net SCHEMA extensions; -- Create supabase_functions schema - CREATE SCHEMA supabase_functions AUTHORIZATION supabase_admin; + CREATE SCHEMA IF NOT EXISTS supabase_functions AUTHORIZATION supabase_admin; GRANT USAGE ON SCHEMA supabase_functions TO postgres, anon, authenticated, service_role; ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON TABLES TO postgres, anon, authenticated, service_role; ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON FUNCTIONS TO postgres, anon, authenticated, service_role; ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON SEQUENCES TO postgres, anon, authenticated, service_role; -- supabase_functions.migrations definition - CREATE TABLE supabase_functions.migrations ( + CREATE TABLE IF NOT EXISTS supabase_functions.migrations ( version text PRIMARY KEY, inserted_at timestamptz NOT NULL DEFAULT NOW() ); -- Initial supabase_functions migration - INSERT INTO supabase_functions.migrations (version) VALUES ('initial'); + INSERT INTO supabase_functions.migrations (version) VALUES ('initial') + ON CONFLICT (version) DO NOTHING; -- supabase_functions.hooks definition - CREATE TABLE supabase_functions.hooks ( + CREATE TABLE IF NOT EXISTS supabase_functions.hooks ( id bigserial PRIMARY KEY, hook_table_id integer NOT NULL, hook_name text NOT NULL, created_at timestamptz NOT NULL DEFAULT NOW(), request_id bigint ); - CREATE INDEX supabase_functions_hooks_request_id_idx ON supabase_functions.hooks USING btree (request_id); - CREATE INDEX supabase_functions_hooks_h_table_id_h_name_idx ON supabase_functions.hooks USING btree (hook_table_id, hook_name); + CREATE INDEX IF NOT EXISTS supabase_functions_hooks_request_id_idx ON supabase_functions.hooks USING btree (request_id); + CREATE INDEX IF NOT EXISTS supabase_functions_hooks_h_table_id_h_name_idx ON supabase_functions.hooks USING btree (hook_table_id, hook_name); COMMENT ON TABLE supabase_functions.hooks IS 'Supabase Functions Hooks: Audit trail for triggered hooks.'; - CREATE FUNCTION supabase_functions.http_request() + CREATE OR REPLACE FUNCTION supabase_functions.http_request() RETURNS trigger LANGUAGE plpgsql AS $function$ @@ -200,9 +201,10 @@ BEGIN; END IF; END $$; - INSERT INTO supabase_functions.migrations (version) VALUES ('20210809183423_update_grants'); + INSERT INTO supabase_functions.migrations (version) VALUES ('20210809183423_update_grants') + ON CONFLICT (version) DO NOTHING; ALTER function supabase_functions.http_request() SECURITY DEFINER; ALTER function supabase_functions.http_request() SET search_path = supabase_functions; REVOKE ALL ON FUNCTION supabase_functions.http_request() FROM PUBLIC; GRANT EXECUTE ON FUNCTION supabase_functions.http_request() TO postgres, anon, authenticated, service_role; -COMMIT; \ No newline at end of file +COMMIT; diff --git a/deploy/tests/test_build_offline_package.sh b/deploy/tests/test_build_offline_package.sh new file mode 100755 index 000000000..791e087ad --- /dev/null +++ b/deploy/tests/test_build_offline_package.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +TMP_DIR="${TMPDIR:-/tmp}/nexent-offline-package-test-$$" +BIN_DIR="$TMP_DIR/bin" +OUT_DIR="$TMP_DIR/out" + +mkdir -p "$BIN_DIR" "$OUT_DIR" +trap 'rm -rf "$TMP_DIR"' EXIT + +fail() { + echo "FAIL: $*" + exit 1 +} + +create_fake_docker() { + cat > "$BIN_DIR/docker" <<'SH' +#!/bin/sh +case "$1" in + image) + if [ "$2" = "inspect" ]; then + [ -n "${FAKE_DOCKER_LOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_DOCKER_LOG" + old_ifs="$IFS" + IFS=',' + for local_image in ${FAKE_DOCKER_LOCAL_IMAGES:-}; do + if [ "$local_image" = "$3" ]; then + IFS="$old_ifs" + exit 0 + fi + done + IFS="$old_ifs" + exit 1 + fi + exit 0 + ;; + pull) + [ -n "${FAKE_DOCKER_LOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_DOCKER_LOG" + exit 0 + ;; + save) + [ -n "${FAKE_DOCKER_LOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_DOCKER_LOG" + out="" + while [ "$#" -gt 0 ]; do + if [ "$1" = "-o" ]; then + out="$2" + shift 2 + continue + fi + shift + done + [ -n "$out" ] && : > "$out" + exit 0 + ;; + *) + exit 0 + ;; +esac +SH + chmod +x "$BIN_DIR/docker" +} + +assert_common_package_files() { + local package_dir="$1" + [ -f "$package_dir/deploy.sh" ] || fail "deploy.sh should be packaged" + [ -f "$package_dir/uninstall.sh" ] || fail "uninstall.sh should be packaged" + [ ! -f "$package_dir/install.sh" ] || fail "install.sh should not be packaged" + [ ! -f "$package_dir/offline-install.sh" ] || fail "offline-install.sh should not be packaged" + [ -f "$package_dir/load-images.sh" ] || fail "load-images.sh should be packaged" + [ -f "$package_dir/manifest.yaml" ] || fail "manifest.yaml should be packaged" + [ -f "$package_dir/checksums.txt" ] || fail "checksums.txt should be packaged" + [ -f "$package_dir/deploy/deploy.sh" ] || fail "deploy/deploy.sh should be packaged" + [ -f "$package_dir/deploy/uninstall.sh" ] || fail "deploy/uninstall.sh should be packaged" + [ -f "$package_dir/VERSION" ] || fail "root VERSION should be packaged" + [ -f "$package_dir/.env.example" ] || fail "root .env.example should be packaged" + [ -f "$package_dir/deploy/sql/init.sql" ] || fail "deploy/sql/init.sql should be packaged" + [ -d "$package_dir/deploy/sql/migrations" ] || fail "deploy/sql/migrations should be packaged" + [ -d "$package_dir/deploy/sql/supabase" ] || fail "deploy/sql/supabase should be packaged" + [ -f "$package_dir/deploy/sql/supabase/webhooks.sql" ] || fail "deploy/sql/supabase/webhooks.sql should be packaged" + [ ! -f "$package_dir/.env" ] || fail ".env should not be packaged" + [ ! -f "$package_dir/deploy/docker/.env" ] || fail "deploy/docker/.env should not be packaged" + [ ! -f "$package_dir/deploy/docker/.env.generated" ] || fail "deploy/docker/.env.generated should not be packaged" + [ ! -f "$package_dir/deploy/docker/deploy.options" ] || fail "deploy/docker/deploy.options should not be packaged" +} + +create_fake_docker + +for target in docker k8s all; do + package_dir="$OUT_DIR/$target" + PATH="$BIN_DIR:$PATH" \ + bash "$PROJECT_ROOT/deploy/offline/build_offline_package.sh" \ + --version v2.2.0 \ + --platform amd64 \ + --components infrastructure,application \ + --image-source general \ + --target "$target" \ + --compress true \ + --output-dir "$package_dir" >/tmp/nexent-offline-package-${target}.log + + assert_common_package_files "$package_dir" + [ -f "$OUT_DIR/nexent-offline-${target}-amd64-v2.2.0.zip" ] || fail "zip package should be created for target $target" + grep -q "target: \"$target\"" "$package_dir/manifest.yaml" || fail "manifest should record target $target" + grep -q "nexent/nexent:v2.2.0" "$package_dir/manifest.yaml" || fail "manifest should include Nexent image" + + case "$target" in + docker) + [ -f "$package_dir/deploy/docker/deploy.sh" ] || fail "docker package should include deploy/docker/deploy.sh" + [ ! -e "$package_dir/deploy/k8s/deploy.sh" ] || fail "docker package should not include k8s deploy script" + ;; + k8s) + [ -f "$package_dir/deploy/k8s/deploy.sh" ] || fail "k8s package should include deploy/k8s/deploy.sh" + [ ! -e "$package_dir/deploy/docker/deploy.sh" ] || fail "k8s package should not include docker deploy script" + ;; + all) + [ -f "$package_dir/deploy/docker/deploy.sh" ] || fail "all package should include deploy/docker/deploy.sh" + [ -f "$package_dir/deploy/k8s/deploy.sh" ] || fail "all package should include deploy/k8s/deploy.sh" + ;; + esac +done + +deploy_wrapper_dir="$OUT_DIR/deploy-wrapper" +mkdir -p "$deploy_wrapper_dir/deploy" +cp "$PROJECT_ROOT/deploy.sh" "$deploy_wrapper_dir/deploy.sh" +cat > "$deploy_wrapper_dir/load-images.sh" <<'SH' +#!/usr/bin/env bash +printf 'load-images\n' >> "$DEPLOY_WRAPPER_LOG" +SH +chmod +x "$deploy_wrapper_dir/load-images.sh" +cat > "$deploy_wrapper_dir/deploy/deploy.sh" <<'SH' +#!/usr/bin/env bash +printf 'deploy:%s\n' "$*" >> "$DEPLOY_WRAPPER_LOG" +SH +chmod +x "$deploy_wrapper_dir/deploy/deploy.sh" + +deploy_wrapper_log="$TMP_DIR/deploy-wrapper.log" +DEPLOY_WRAPPER_LOG="$deploy_wrapper_log" bash "$deploy_wrapper_dir/deploy.sh" docker --foo bar +if grep -q '^load-images$' "$deploy_wrapper_log"; then + fail "deploy.sh should not load images by default" +fi +grep -q '^deploy:docker --foo bar$' "$deploy_wrapper_log" || fail "deploy.sh should forward args without --load-images" + +: > "$deploy_wrapper_log" +DEPLOY_WRAPPER_LOG="$deploy_wrapper_log" bash "$deploy_wrapper_dir/deploy.sh" --load-images docker --foo bar +first_line="$(sed -n '1p' "$deploy_wrapper_log")" +second_line="$(sed -n '2p' "$deploy_wrapper_log")" +[ "$first_line" = "load-images" ] || fail "deploy.sh --load-images should load images before deploy" +[ "$second_line" = "deploy:docker --foo bar" ] || fail "deploy.sh --load-images should strip only the wrapper flag" + +latest_package_dir="$OUT_DIR/latest" +latest_pull_log="$TMP_DIR/latest-docker.log" +: > "$latest_pull_log" + +PATH="$BIN_DIR:$PATH" FAKE_DOCKER_LOG="$latest_pull_log" \ + bash "$PROJECT_ROOT/deploy/offline/build_offline_package.sh" \ + --version latest \ + --platform amd64 \ + --components infrastructure,application \ + --image-source general \ + --target docker \ + --compress true \ + --output-dir "$latest_package_dir" >/tmp/nexent-offline-package-latest.log + +assert_common_package_files "$latest_package_dir" +[ -f "$OUT_DIR/nexent-offline-docker-amd64-latest.zip" ] || fail "zip package should be created for latest package" +grep -q "nexent/nexent:latest" "$latest_package_dir/manifest.yaml" || fail "manifest should include local latest Nexent image" +! grep -q '^pull .*nexent/nexent:latest$' "$latest_pull_log" || fail "latest Nexent image should not be pulled" +! grep -q '^pull .*nexent/nexent-web:latest$' "$latest_pull_log" || fail "latest Nexent web image should not be pulled" +! grep -q '^pull .*nexent/nexent-mcp:latest$' "$latest_pull_log" || fail "latest Nexent MCP image should not be pulled" +grep -q '^pull .*docker.elastic.co/elasticsearch/elasticsearch:8.17.4$' "$latest_pull_log" || fail "non-latest infrastructure images should still be pulled" + +local_package_dir="$OUT_DIR/local-existing/package" +local_pull_log="$TMP_DIR/local-existing-docker.log" +: > "$local_pull_log" + +PATH="$BIN_DIR:$PATH" \ + FAKE_DOCKER_LOG="$local_pull_log" \ + FAKE_DOCKER_LOCAL_IMAGES="nexent/nexent:v2.2.0,docker.elastic.co/elasticsearch/elasticsearch:8.17.4" \ + bash "$PROJECT_ROOT/deploy/offline/build_offline_package.sh" \ + --version v2.2.0 \ + --platform amd64 \ + --components infrastructure,application \ + --image-source general \ + --target docker \ + --compress true \ + --output-dir "$local_package_dir" >/tmp/nexent-offline-package-local-existing.log + +assert_common_package_files "$local_package_dir" +[ -f "$OUT_DIR/local-existing/nexent-offline-docker-amd64-v2.2.0.zip" ] || fail "zip package should be created for local existing package" +! grep -q '^pull .*nexent/nexent:v2.2.0$' "$local_pull_log" || fail "existing local Nexent image should not be pulled" +! grep -q '^pull .*docker.elastic.co/elasticsearch/elasticsearch:8.17.4$' "$local_pull_log" || fail "existing local infrastructure image should not be pulled" +grep -q '^pull .*nexent/nexent-web:v2.2.0$' "$local_pull_log" || fail "missing non-latest Nexent web image should still be pulled" + +default_package_dir="$OUT_DIR/default-no-compress/package" +PATH="$BIN_DIR:$PATH" \ + bash "$PROJECT_ROOT/deploy/offline/build_offline_package.sh" \ + --version v2.2.0 \ + --platform amd64 \ + --components infrastructure,application \ + --image-source general \ + --target docker \ + --output-dir "$default_package_dir" >/tmp/nexent-offline-package-default-no-compress.log + +assert_common_package_files "$default_package_dir" +[ ! -f "$OUT_DIR/default-no-compress/nexent-offline-docker-amd64-v2.2.0.zip" ] || fail "zip package should not be created by default" + +echo "All offline package tests passed." diff --git a/deploy/tests/test_common.sh b/deploy/tests/test_common.sh new file mode 100755 index 000000000..21245ae9d --- /dev/null +++ b/deploy/tests/test_common.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "$SCRIPT_DIR/../common/common.sh" +# shellcheck source=/dev/null +source "$SCRIPT_DIR/../common/version.sh" + +TMP_DIR="${TMPDIR:-/tmp}/nexent-deployment-test-$$" +mkdir -p "$TMP_DIR" +trap 'rm -rf "$TMP_DIR"' EXIT + +assert_eq() { + local expected="$1" + local actual="$2" + local message="$3" + if [ "$expected" != "$actual" ]; then + echo "FAIL: $message" + echo " expected: $expected" + echo " actual: $actual" + exit 1 + fi +} + +assert_contains() { + local haystack="$1" + local needle="$2" + local message="$3" + if [[ "$haystack" != *"$needle"* ]]; then + echo "FAIL: $message" + echo " missing: $needle" + echo " in: $haystack" + exit 1 + fi +} + +assert_success() { + local message="$1" + shift + if ! "$@"; then + echo "FAIL: $message" + exit 1 + fi +} + +write_full_config() { + local file="$1" + { + echo 'schemaVersion: "1"' + echo 'appVersion: "latest"' + echo 'components:' + echo ' - infrastructure' + echo ' - application' + echo ' - data-process' + echo ' - supabase' + echo ' - terminal' + echo 'portPolicy: "development"' + echo 'imageSource: "local-latest"' + } > "$file" +} + +APP_VERSION="latest" +deployment_prepare_config --app-version latest +assert_eq "infrastructure,application,data-process,supabase" "$DEPLOYMENT_COMPONENTS" "default components should include data-process and supabase" +assert_contains "$DEPLOYMENT_SELECTED_DOCKER_SERVICES" "nexent-data-process" "default docker services should include data-process" +assert_contains "$DEPLOYMENT_SELECTED_HELM_CHARTS" "nexent-supabase-db" "default helm charts should include supabase db" +deployment_prepare_config --components infrastructure,application --port-policy production --image-source general --app-version latest +assert_eq "infrastructure,application" "$DEPLOYMENT_COMPONENTS" "components should come from CLI" +assert_eq "production" "$DEPLOYMENT_PORT_POLICY" "port policy should come from CLI" +assert_eq "general" "$DEPLOYMENT_IMAGE_SOURCE" "image source should come from CLI" +assert_contains "$DEPLOYMENT_SELECTED_DOCKER_SERVICES" "nexent-web" "application services should include web" +if [[ "$DEPLOYMENT_SELECTED_DOCKER_SERVICES" == *"nexent-data-process"* ]]; then + echo "FAIL: application should not include data-process" + exit 1 +fi +assert_contains "$DEPLOYMENT_DOCKER_PORTS" "3000" "production should expose web" + +deployment_prepare_config --components supabase --port-policy development --app-version latest +assert_eq "infrastructure,supabase" "$DEPLOYMENT_COMPONENTS" "only infrastructure should be required and added" +if [[ "$DEPLOYMENT_SELECTED_DOCKER_SERVICES" == *"nexent-web"* ]]; then + echo "FAIL: application should not be auto-added" + exit 1 +fi + +deployment_prepare_config --components infrastructure,application --port-policy development --registry-profile mainland --app-version latest +assert_eq "mainland" "$DEPLOYMENT_IMAGE_SOURCE" "legacy registry profile should map to mainland image source" + +if deployment_prepare_config --components infrastructure,application --port-policy development --image-source pinned --app-version latest 2>/dev/null; then + echo "FAIL: pinned image source should be rejected" + exit 1 +fi + +DEPLOYMENT_VERSION="full" +DEPLOYMENT_MODE="development" +IS_MAINLAND="Y" +deployment_prepare_config --app-version latest +assert_contains "$DEPLOYMENT_COMPONENTS" "supabase" "legacy full should include supabase" +assert_eq "mainland" "$DEPLOYMENT_REGISTRY_PROFILE" "legacy mainland flag should map registry profile" +assert_eq "mainland" "$DEPLOYMENT_IMAGE_SOURCE" "legacy mainland flag should map image source" +unset DEPLOYMENT_VERSION DEPLOYMENT_MODE IS_MAINLAND + +FULL_CONFIG="$TMP_DIR/full.yaml" +write_full_config "$FULL_CONFIG" +deployment_prepare_config --config "$FULL_CONFIG" +deployment_apply_image_source +assert_eq "nexent/nexent:latest" "$NEXENT_IMAGE" "local-latest image should be applied" +assert_contains "$DEPLOYMENT_SELECTED_HELM_CHARTS" "nexent-data-process" "data-process chart should be selected" + +DEPLOYMENT_VERSION="speed" +DEPLOYMENT_MODE="production" +IS_MAINLAND="Y" +deployment_prepare_config --local-config "$FULL_CONFIG" --use-local-config --app-version latest +assert_contains "$DEPLOYMENT_COMPONENTS" "data-process" "use local config should keep saved data-process when legacy env exists" +assert_contains "$DEPLOYMENT_SELECTED_DOCKER_SERVICES" "nexent-data-process" "use local config should select data-process docker service" +assert_eq "development" "$DEPLOYMENT_PORT_POLICY" "use local config should keep saved port policy over legacy mode" +assert_eq "local-latest" "$DEPLOYMENT_IMAGE_SOURCE" "use local config should keep saved image source over legacy mainland flag" +unset DEPLOYMENT_VERSION DEPLOYMENT_MODE IS_MAINLAND + +LOCAL_HELM_VALUES="$TMP_DIR/local-generated-values.yaml" +deployment_render_helm_values "$LOCAL_HELM_VALUES" +assert_contains "$(sed -n '1,90p' "$LOCAL_HELM_VALUES")" "repository: \"nexent/nexent\"" "local-latest should render mcp chart with backend image" +assert_contains "$(sed -n '1,90p' "$LOCAL_HELM_VALUES")" "pullPolicy: \"Never\"" "local-latest should render mcp chart with local pull policy" +assert_contains "$(sed -n '140,180p' "$LOCAL_HELM_VALUES")" "repository: \"nexent/nexent-mcp\"" "local-latest should keep common mcp docker image" + +DEPLOYMENT_VERSION="speed" +deployment_prepare_config --local-config "$FULL_CONFIG" --reconfigure --image-source general --app-version latest +assert_eq "false" "$DEPLOYMENT_CONFIG_FILE_LOADED" "reconfigure should use local config as defaults without skipping configuration" +assert_contains "$DEPLOYMENT_COMPONENTS" "data-process" "reconfigure defaults should include saved components" +assert_eq "development" "$DEPLOYMENT_PORT_POLICY" "reconfigure defaults should include saved port policy" +assert_eq "general" "$DEPLOYMENT_IMAGE_SOURCE" "explicit image source should override reconfigure defaults" +unset DEPLOYMENT_VERSION + +HELM_VALUES="$TMP_DIR/generated-values.yaml" +deployment_render_helm_values "$HELM_VALUES" +assert_contains "$(sed -n '1,220p' "$HELM_VALUES")" "data-process: true" "component table should include data-process" +assert_contains "$(sed -n '1,260p' "$HELM_VALUES")" "type: \"NodePort\"" "development policy should render NodePort values" +assert_contains "$(sed -n '1,260p' "$HELM_VALUES")" "enabled: true" "selected charts should be enabled" + +DOCKER_ENV="$TMP_DIR/.env.generated" +deployment_render_docker_env "$DOCKER_ENV" +assert_contains "$(sed -n '1,120p' "$DOCKER_ENV")" "NEXENT_IMAGE=" "docker generated env should contain image variables" +if grep -Eq '^DEPLOYMENT_(SCHEMA_VERSION|COMPONENTS|PORT_POLICY|IMAGE_SOURCE|REGISTRY_PROFILE|APP_VERSION|MONITORING_PROVIDER|SELECTED_DOCKER_SERVICES|DOCKER_PORTS)=' "$DOCKER_ENV"; then + echo "FAIL: docker generated env should not contain persisted deployment decisions" + exit 1 +fi + +LOCAL_CONFIG="$TMP_DIR/local-config.yaml" +deployment_persist_local_config "$LOCAL_CONFIG" +if grep -Eq 'PASSWORD|TOKEN|JWT|SECRET|KEY' "$LOCAL_CONFIG"; then + echo "FAIL: persisted local config should not contain secret-looking fields" + exit 1 +fi +if grep -q 'registryProfile' "$LOCAL_CONFIG"; then + echo "FAIL: persisted local config should not contain registryProfile" + exit 1 +fi + +assert_success "b should be treated as TUI back key" deployment_tui_is_back_key "b" +assert_success "Backspace should be treated as TUI back key" deployment_tui_is_back_key $'\177' +if deployment_tui_is_back_key "q"; then + echo "FAIL: q should remain the TUI quit key" + exit 1 +fi + +deployment_tui_step_should_run() { + case "$1" in + 0|1|2) + return 0 + ;; + 3) + return 1 + ;; + esac + return 1 +} +assert_eq "1" "$(deployment_tui_next_step 0)" "TUI next step should advance to the next runnable step" +assert_eq "4" "$(deployment_tui_next_step 2)" "TUI next step should skip non-runnable monitoring provider" +assert_eq "2" "$(deployment_tui_previous_step 3)" "TUI previous step should skip non-runnable steps" + +assert_eq "$(sed -n '1p' "$SCRIPT_DIR/../../VERSION")" "$(deployment_read_version "")" "deployment version should come from root VERSION" +assert_eq "v-test" "$(deployment_read_version "v-test")" "explicit deployment version should win" + +assert_success "password validation should accept frontend-compatible passwords" deployment_validate_password "Nexent123" +if deployment_validate_password "nexent123"; then + echo "FAIL: password without uppercase letters should be rejected" + exit 1 +fi +if deployment_validate_password "NEXENT123"; then + echo "FAIL: password without lowercase letters should be rejected" + exit 1 +fi +if deployment_validate_password "NexentPwd"; then + echo "FAIL: password without numbers should be rejected" + exit 1 +fi +if deployment_validate_password "Nex123"; then + echo "FAIL: password shorter than 8 characters should be rejected" + exit 1 +fi + +ENV_TEST_ROOT="$TMP_DIR/env-root" +mkdir -p "$ENV_TEST_ROOT/docker" +printf 'FROM_DOCKER=yes\n' > "$ENV_TEST_ROOT/docker/.env" +printf 'FROM_EXAMPLE=yes\n' > "$ENV_TEST_ROOT/.env.example" +deployment_ensure_root_env "$ENV_TEST_ROOT" "$ENV_TEST_ROOT/docker" +assert_contains "$(cat "$ENV_TEST_ROOT/.env")" "FROM_DOCKER=yes" "root .env should migrate from docker/.env first" + +printf 'ROOT_ONLY=yes\n' > "$ENV_TEST_ROOT/.env" +deployment_ensure_root_env "$ENV_TEST_ROOT" "$ENV_TEST_ROOT/docker" +assert_contains "$(cat "$ENV_TEST_ROOT/.env")" "ROOT_ONLY=yes" "existing root .env should not be overwritten" + +deployment_update_env_var_file "$ENV_TEST_ROOT/.env" "ROOT_ONLY" "updated" +assert_contains "$(cat "$ENV_TEST_ROOT/.env")" 'ROOT_ONLY="updated"' "env updater should update root env values" +assert_eq "true" "$DEPLOYMENT_LAST_ENV_WRITE_CHANGED" "env updater should mark changed writes" + +ENV_CONTENT_BEFORE="$(cat "$ENV_TEST_ROOT/.env")" +deployment_update_env_var_file "$ENV_TEST_ROOT/.env" "ROOT_ONLY" "updated" +assert_eq "false" "$DEPLOYMENT_LAST_ENV_WRITE_CHANGED" "env updater should mark identical writes unchanged" +assert_eq "$ENV_CONTENT_BEFORE" "$(cat "$ENV_TEST_ROOT/.env")" "env updater should not rewrite identical quoted values" + +printf 'UNQUOTED=value\nSINGLE_QUOTED='\''value2'\''\n' >> "$ENV_TEST_ROOT/.env" +assert_eq "value" "$(deployment_get_env_var_file "$ENV_TEST_ROOT/.env" "UNQUOTED")" "env getter should read unquoted values" +assert_eq "value2" "$(deployment_get_env_var_file "$ENV_TEST_ROOT/.env" "SINGLE_QUOTED")" "env getter should read single-quoted values" +deployment_update_env_var_file "$ENV_TEST_ROOT/.env" "UNQUOTED" "value" +assert_eq "false" "$DEPLOYMENT_LAST_ENV_WRITE_CHANGED" "env updater should normalize unquoted identical values" + +GENERATE_ENV_TEST_ROOT="$TMP_DIR/generate-env-root" +mkdir -p "$GENERATE_ENV_TEST_ROOT/docker" +printf 'FROM_GENERATE_DOCKER=yes\n' > "$GENERATE_ENV_TEST_ROOT/docker/.env" +printf 'FROM_GENERATE_EXAMPLE=yes\n' > "$GENERATE_ENV_TEST_ROOT/.env.example" +( + NEXENT_GENERATE_ENV_SKIP_MAIN=true + # shellcheck source=/dev/null + source "$SCRIPT_DIR/../docker/generate_env.sh" + ENV_FILE="$GENERATE_ENV_TEST_ROOT/.env" + ENV_EXAMPLE="$GENERATE_ENV_TEST_ROOT/.env.example" + LEGACY_ENV="$GENERATE_ENV_TEST_ROOT/docker/.env" + LEGACY_ENV_EXAMPLE="$GENERATE_ENV_TEST_ROOT/docker/.env.example" + prepare_env_file >/dev/null +) +assert_contains "$(cat "$GENERATE_ENV_TEST_ROOT/.env")" "FROM_GENERATE_DOCKER=yes" "generate_env should migrate docker/.env before .env.example" +echo "All deployment common tests passed." diff --git a/deploy/tests/test_images_build.sh b/deploy/tests/test_images_build.sh new file mode 100755 index 000000000..eb1310867 --- /dev/null +++ b/deploy/tests/test_images_build.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +BUILD_SCRIPT="$PROJECT_ROOT/deploy/images/build.sh" + +fail() { + echo "FAIL: $*" + exit 1 +} + +assert_contains() { + local haystack="$1" + local needle="$2" + local message="$3" + if [[ "$haystack" != *"$needle"* ]]; then + echo "FAIL: $message" + echo " missing: $needle" + echo " in: $haystack" + exit 1 + fi +} + +assert_not_contains() { + local haystack="$1" + local needle="$2" + local message="$3" + if [[ "$haystack" == *"$needle"* ]]; then + echo "FAIL: $message" + echo " unexpected: $needle" + echo " in: $haystack" + exit 1 + fi +} + +output="$(bash "$BUILD_SCRIPT" --images main,web,mcp,data-process --version latest --registry general --dry-run)" +assert_contains "$output" "nexent/nexent:latest" "image list should build main image with latest tag" +assert_contains "$output" "nexent/nexent-web:latest" "image list should build web image with latest tag" +assert_contains "$output" "nexent/nexent-mcp:latest" "image list should build mcp image with latest tag" +assert_contains "$output" "nexent/nexent-data-process:latest" "image list should build data-process image with latest tag" +assert_not_contains "$output" "nexent/nexent-ubuntu-terminal:latest" "terminal image should not be built when terminal image is absent" +assert_not_contains "$output" "--platform" "default build should use local architecture" + +output="$(bash "$BUILD_SCRIPT" --main --version latest --platform linux/amd64 --dry-run)" +assert_contains "$output" "--platform linux/amd64" "explicit platform should be forwarded" +assert_contains "$output" "nexent/nexent:latest" "explicit platform build should still build selected image" + +output="$(bash "$BUILD_SCRIPT" --terminal --version v9.9.9 --registry mainland --dry-run)" +assert_contains "$output" "ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:v9.9.9" "terminal option should build terminal image with selected version" +assert_not_contains "$output" "ccr.ccs.tencentyun.com/nexent-hub/nexent:v9.9.9" "main image should not be built for terminal-only option" + +output="$(bash "$BUILD_SCRIPT" --web --docs --version v8.8.8 --registry general --dry-run)" +assert_contains "$output" "nexent/nexent-web:v8.8.8" "web option should build web image" +assert_contains "$output" "nexent/nexent-docs:v8.8.8" "docs option should build docs image" +assert_not_contains "$output" "nexent/nexent-data-process:v8.8.8" "data-process image should not be built when option is absent" + +output="$(bash "$BUILD_SCRIPT" --image web --version v1.2.3 --registry general --dry-run)" +assert_contains "$output" "nexent/nexent-web:v1.2.3" "explicit image build should keep supporting selected versions" +assert_not_contains "$output" "nexent/nexent:v1.2.3" "single image build should not build main image" + +output="$(bash "$BUILD_SCRIPT" --components infrastructure,supabase,monitoring --version latest --dry-run)" +assert_contains "$output" "No Nexent images selected for build." "legacy non-application components should produce no Nexent image builds" + +if bash "$BUILD_SCRIPT" --images main,unknown --dry-run >/tmp/nexent-image-build-invalid.log 2>&1; then + fail "unknown image should fail" +fi +assert_contains "$(cat /tmp/nexent-image-build-invalid.log)" "Unsupported image: unknown" "unknown image should explain the error" + +if bash "$BUILD_SCRIPT" --data-process --variant slim --dry-run >/tmp/nexent-image-build-variant.log 2>&1; then + fail "deprecated data-process variant option should fail" +fi +assert_contains "$(cat /tmp/nexent-image-build-variant.log)" "Unknown option: --variant" "deprecated data-process variant option should be rejected" + +output="$( + printf 'main,web,mcp,data-process\n1\n1\n' | \ + bash "$BUILD_SCRIPT" --interactive --dry-run +)" +assert_contains "$output" "Nexent image build configuration" "interactive mode should show configuration prompt" +assert_contains "$output" "nexent/nexent:latest" "interactive mode should accept latest version selection" +assert_contains "$output" "nexent/nexent-web:latest" "interactive image selection should include web image" +assert_contains "$output" "nexent/nexent-mcp:latest" "interactive image selection should include mcp image" +assert_contains "$output" "nexent/nexent-data-process:latest" "interactive image selection should include data-process image" +assert_not_contains "$output" "nexent/nexent-ubuntu-terminal:latest" "interactive image selection should exclude unselected terminal image" +assert_not_contains "$output" "--platform" "interactive mode should use local architecture by default" + +output="$( + printf '\n\n1\n' | \ + bash "$BUILD_SCRIPT" --interactive --dry-run +)" +assert_contains "$output" "nexent/nexent:latest" "interactive default image selection should include main image" +assert_contains "$output" "nexent/nexent-web:latest" "interactive default image selection should include web image" +assert_not_contains "$output" "nexent/nexent-mcp:latest" "interactive default image selection should not include mcp image" +assert_not_contains "$output" "nexent/nexent-data-process:latest" "interactive default image selection should not include data-process image" +assert_not_contains "$output" "nexent/nexent-ubuntu-terminal:latest" "interactive default image selection should not include terminal image" + +echo "All image build tests passed." diff --git a/deploy/tests/test_sql_migrations.sh b/deploy/tests/test_sql_migrations.sh new file mode 100755 index 000000000..c8622009d --- /dev/null +++ b/deploy/tests/test_sql_migrations.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOY_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +MIGRATION_SCRIPT="$DEPLOY_ROOT/common/run-sql-migrations.sh" +TMP_DIR="${TMPDIR:-/tmp}/nexent-sql-migration-test-$$" +SQL_DIR="$TMP_DIR/sql/migrations" +BIN_DIR="$TMP_DIR/bin" + +mkdir -p "$SQL_DIR" "$BIN_DIR" +trap 'rm -rf "$TMP_DIR"' EXIT + +fail() { + echo "FAIL: $*" + exit 1 +} + +assert_file_contains() { + local file="$1" + local needle="$2" + local message="$3" + if ! grep -Fq "$needle" "$file"; then + fail "$message" + fi +} + +assert_file_not_contains() { + local file="$1" + local needle="$2" + local message="$3" + if grep -Fq "$needle" "$file"; then + fail "$message" + fi +} + +create_fake_psql() { + cat > "$BIN_DIR/psql" <<'SH' +#!/bin/sh +prev="" +capture_next_query=false +for arg in "$@"; do + if [ "$prev" = "-f" ]; then + if [ -n "$CAPTURE_PLAN" ]; then + cp "$arg" "$CAPTURE_PLAN" + fi + exit 0 + fi + if [ "$prev" = "-c" ] || [ "$capture_next_query" = true ]; then + if [ -n "$CAPTURE_QUERY" ]; then + printf '%s\n' "$arg" >> "$CAPTURE_QUERY" + fi + case "$arg" in + "SELECT 1") + printf '1\n' + ;; + *) + printf '%s\n' "${FAKE_WAIT_STATUS:-ready}" + ;; + esac + exit 0 + fi + case "$arg" in + -*c*) + capture_next_query=true + ;; + esac + prev="$arg" +done +cat >/dev/null +exit 0 +SH + chmod +x "$BIN_DIR/psql" +} + +create_fake_psql + +cat > "$SQL_DIR/v1_merged_migrations.sql" <<'SQL' +CREATE TABLE IF NOT EXISTS nexent.test_table(id int); +ALTER TABLE nexent.test_table ADD COLUMN IF NOT EXISTS name text; +SQL +cat > "$SQL_DIR/v2_test.sql" <<'SQL' +CREATE TABLE IF NOT EXISTS nexent.test_table_v2(id int); +SQL + +SYMLINK_SQL_DIR="$TMP_DIR/sql/migrations-link" +ln -s "$SQL_DIR" "$SYMLINK_SQL_DIR" 2>/dev/null || cp -R "$SQL_DIR" "$SYMLINK_SQL_DIR" + +INIT_SQL_FILE="$TMP_DIR/init.sql" +printf 'create schema if not exists nexent;\ncreate table if not exists nexent.model_record_t(id int);\ncreate table if not exists nexent.knowledge_record_t(id int);\ncreate table if not exists nexent.ag_tenant_agent_t(id int);\ncreate table if not exists nexent.conversation_record_t(id int);\ncreate table if not exists nexent.conversation_message_t(id int);\ncreate table if not exists nexent.ag_tool_info_t(id int);\n' > "$INIT_SQL_FILE" + +if grep -Eq '^COMMENT ON COLUMN nexent\.ag_tenant_agent_t\.prompt ' "$DEPLOY_ROOT/sql/init.sql"; then + fail "init SQL should not comment ag_tenant_agent_t.prompt because a later migration drops that column" +fi +if grep -Eq '^COMMENT ON COLUMN nexent\.model_record_t\.is_deep_thinking ' "$DEPLOY_ROOT/sql/init.sql"; then + fail "init SQL should not comment model_record_t.is_deep_thinking because a later migration drops that column" +fi + +PLAN_FILE="$TMP_DIR/plan.sql" +PATH="$BIN_DIR:$PATH" \ +CAPTURE_PLAN="$PLAN_FILE" \ +CAPTURE_QUERY="" \ +NEXENT_SQL_INIT_FILE="$INIT_SQL_FILE" \ +NEXENT_SQL_MIGRATION_DIR="$SYMLINK_SQL_DIR" \ +NEXENT_SQL_WAIT_TIMEOUT_SECONDS=1 \ +NEXENT_APP_VERSION="v-test" \ + bash "$MIGRATION_SCRIPT" --migrate >/tmp/nexent-sql-migration-test.log + +[ -f "$PLAN_FILE" ] || fail "migration plan should be captured" +assert_file_contains "$PLAN_FILE" "pg_advisory_lock" "plan should acquire advisory lock" +assert_file_contains "$PLAN_FILE" "pg_advisory_unlock" "plan should release advisory lock" +assert_file_contains "$PLAN_FILE" "status text NOT NULL DEFAULT 'applied'" "plan should create extended migration table status" +assert_file_contains "$PLAN_FILE" "app_version text" "plan should create app_version field" +assert_file_contains "$PLAN_FILE" "source_file text" "plan should create source_file field" +assert_file_contains "$PLAN_FILE" "CHECK (status IN ('applied', 'baselined'))" "plan should keep compatibility with prior baselined records" +assert_file_not_contains "$PLAN_FILE" "_nexent_migration_probe_result" "plan should not use probe temp tables" +assert_file_not_contains "$PLAN_FILE" "nexent-migration-probe" "plan should not require SQL marker comments" +assert_file_contains "$PLAN_FILE" "\\i '$INIT_SQL_FILE'" "plan should always apply init SQL" +assert_file_contains "$PLAN_FILE" "VALUES ('__init.sql'" "plan should record init SQL" +assert_file_contains "$PLAN_FILE" "'applied', 'v-test'" "plan should record applied status and app version" +assert_file_contains "$PLAN_FILE" "ON CONFLICT (migration_id) DO UPDATE SET" "plan should update migration records after execution" +assert_file_contains "$PLAN_FILE" "\\echo [sql-migrations] check v1_merged_migrations.sql" "plan should check migrations by file name" +assert_file_contains "$PLAN_FILE" "\\echo [sql-migrations] skip v1_merged_migrations.sql" "plan should skip matching checksums" +assert_file_contains "$PLAN_FILE" "\\echo [sql-migrations] apply v1_merged_migrations.sql" "plan should apply new migration files" +assert_file_contains "$PLAN_FILE" "\\echo [sql-migrations] reapply v1_merged_migrations.sql" "plan should reapply changed migration files" +assert_file_contains "$PLAN_FILE" "migration_checksum_matched" "plan should compare recorded checksum with current file checksum" +assert_file_contains "$PLAN_FILE" "executed_at = now()" "plan should refresh execution time on reapply" +assert_file_contains "$PLAN_FILE" "SET search_path TO \"nexent\", public;" "plan should set search path for legacy migrations" + +first_check="$(grep -nF '\echo [sql-migrations] check v' "$PLAN_FILE" | head -1 | cut -d: -f2-)" +[ "$first_check" = "\\echo [sql-migrations] check v1_merged_migrations.sql" ] || fail "migrations should be sorted before execution" + +WAIT_QUERY_FILE="$TMP_DIR/wait-query.sql" +WAIT_TABLE_PLAN="$TMP_DIR/wait-table-plan.sql" +PATH="$BIN_DIR:$PATH" \ +CAPTURE_PLAN="$WAIT_TABLE_PLAN" \ +CAPTURE_QUERY="$WAIT_QUERY_FILE" \ +FAKE_WAIT_STATUS="ready" \ +NEXENT_SQL_INIT_FILE="$INIT_SQL_FILE" \ +NEXENT_SQL_MIGRATION_DIR="$SYMLINK_SQL_DIR" \ +NEXENT_SQL_WAIT_TIMEOUT_SECONDS=1 \ + bash "$MIGRATION_SCRIPT" --wait >/tmp/nexent-sql-migration-wait-test.log + +[ -f "$WAIT_TABLE_PLAN" ] || fail "wait mode should ensure migration table" +[ -f "$WAIT_QUERY_FILE" ] || fail "wait mode should query migration target state" +assert_file_contains "$WAIT_QUERY_FILE" "__init.sql" "wait query should include init migration target" +assert_file_contains "$WAIT_QUERY_FILE" "v1_merged_migrations.sql" "wait query should include file-name migration target" +assert_file_contains "$WAIT_QUERY_FILE" "v2_test.sql" "wait query should include all migration files" +assert_file_contains "$WAIT_QUERY_FILE" "actual_checksum = expected_checksum" "wait query should wait for current checksums" +assert_file_contains "$WAIT_QUERY_FILE" "status IN ('applied', 'baselined')" "wait query should accept applied and prior baselined records" +assert_file_not_contains "$WAIT_QUERY_FILE" "checksum_mismatch" "wait mode should allow migrator to reapply checksum changes" + +if grep -R -n '^-- nexent-migration-' "$DEPLOY_ROOT/sql/migrations" --include='*.sql' >/tmp/nexent-sql-marker-check.log; then + cat /tmp/nexent-sql-marker-check.log + fail "migration SQL files should not contain nexent-migration marker comments" +fi + +if grep -R -n 'nexent-migration-' "$DEPLOY_ROOT/common/run-sql-migrations.sh" >/tmp/nexent-runner-marker-check.log; then + cat /tmp/nexent-runner-marker-check.log + fail "migration runner should not parse nexent-migration marker comments" +fi + +echo "All SQL migration tests passed." diff --git a/deploy/uninstall.sh b/deploy/uninstall.sh new file mode 100755 index 000000000..01632236c --- /dev/null +++ b/deploy/uninstall.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +usage() { + cat <<'USAGE' +Usage: + bash uninstall.sh docker [docker uninstall options] + bash uninstall.sh k8s [k8s uninstall options] + +Docker implementation: deploy/docker/uninstall.sh +K8s implementation: deploy/k8s/uninstall.sh +USAGE +} + +case "${1:-}" in + docker) + shift + exec bash "$SCRIPT_DIR/docker/uninstall.sh" "$@" + ;; + k8s|kubernetes|helm) + shift + exec bash "$SCRIPT_DIR/k8s/uninstall.sh" "$@" + ;; + --help|-h|"") + usage + ;; + *) + echo "Unknown uninstall target: $1" >&2 + usage >&2 + exit 1 + ;; +esac diff --git a/doc/docs/en/deployment/devcontainer.md b/doc/docs/en/deployment/devcontainer.md index ce6efe7be..ce62d9fbf 100644 --- a/doc/docs/en/deployment/devcontainer.md +++ b/doc/docs/en/deployment/devcontainer.md @@ -25,8 +25,8 @@ This development container configuration sets up a complete Nexent development e 1. Clone the project locally 2. Open project folder in Cursor/VS Code -3. Run `./deploy.sh --components infrastructure,application --port-policy development` from the `docker` directory to start base containers -4. Enter `nexent-minio` and `nexent-elasticsearch` containers, copy `MINIO_ACCESS_KEY`, `MINIO_SECRET_KEY`, `ELASTICSEARCH_API_KEY` environment variables to corresponding positions in `docker/docker-compose.dev.yml` +3. Run `bash deploy.sh docker --components infrastructure,application,data-process,supabase --port-policy development` from the repository root to start base containers +4. Enter `nexent-minio` and `nexent-elasticsearch` containers, copy `MINIO_ACCESS_KEY`, `MINIO_SECRET_KEY`, `ELASTICSEARCH_API_KEY` environment variables to corresponding positions in `deploy/docker/compose/docker-compose.dev.yml` 5. Press `F1` or `Ctrl+Shift+P`, type `Dev Containers: Reopen in Container ...` 6. Cursor will start the development container based on configuration in `.devcontainer` directory @@ -54,7 +54,7 @@ The following ports are mapped in devcontainer.json: You can customize the development environment by modifying: - `.devcontainer/devcontainer.json` - Plugin configuration -- `docker/docker-compose.dev.yml` - Development container build configuration, requires environment variable modification for proper startup +- `deploy/docker/compose/docker-compose.dev.yml` - Development container build configuration, requires environment variable modification for proper startup ## 5. Troubleshooting diff --git a/doc/docs/en/deployment/docker-build.md b/doc/docs/en/deployment/docker-build.md index bf36dc5d4..a69856606 100644 --- a/doc/docs/en/deployment/docker-build.md +++ b/doc/docs/en/deployment/docker-build.md @@ -1,54 +1,104 @@ ### 🏗️ Build and Push Images +Recommended unified build entry: + +```bash +# Run interactive selection, similar to the deploy scripts +bash deploy/images/build.sh + +# Build selected images with a fixed version tag +bash deploy/images/build.sh \ + --images main,web,mcp,data-process,terminal \ + --version v2.2.1 \ + --registry general \ + --platform linux/amd64,linux/arm64 \ + --push + +# Build the same image set as latest +bash deploy/images/build.sh \ + --images main,web,mcp,data-process \ + --version latest \ + --registry general \ + --platform linux/amd64 \ + --load + +# Build one or more explicit images when needed +bash deploy/images/build.sh --web --docs --version v2.2.1 --dry-run +``` + +When run in a terminal without arguments, `deploy/images/build.sh` prompts for images, image version (`latest` or root `VERSION`), and registry. The interactive defaults are images `main,web` and version `latest`. Use `--interactive` to force the same prompts. + +`--platform` is command-line only. Omit it to build for the local architecture. + +Variant options: +- `--dependency-variant cpu|gpu` controls data-process dependencies and defaults to `cpu`. `gpu` builds GPU/CUDA dependencies and uses the `-gpu` image-name suffix. +- `--terminal-variant slim|conda` controls the terminal image and defaults to `slim`. `conda` keeps Miniconda, `vim`, and the compiler toolchain and uses the `-conda` image-name suffix. + +When building `data-process`, `deploy/images/build.sh` prepares `model-assets` automatically: it first uses an existing root `model-assets` directory, then tries `~/model-assets`, and otherwise clones the Hugging Face repository and runs `git lfs pull`. If you run `docker build` directly, prepare `model-assets` in the repository root first. + +Image options: +- `--main` builds `nexent` +- `--web` builds `nexent-web` +- `--data-process` builds `nexent-data-process` +- `--mcp` builds `nexent-mcp` +- `--terminal` builds `nexent-ubuntu-terminal` +- `--docs` builds `nexent-docs` + ```bash # 🛠️ Create and use a new builder instance that supports multi-architecture builds docker buildx create --name nexent_builder --use # 🚀 build application for multiple architectures -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent -f make/main/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent -f make/web/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent -f deploy/images/dockerfiles/main/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent -f deploy/images/dockerfiles/web/Dockerfile . --push # 📊 build data_process for multiple architectures -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-data-process -f make/data_process/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process -f make/web/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-data-process -f deploy/images/dockerfiles/data-process/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process -f deploy/images/dockerfiles/web/Dockerfile . --push # 🌐 build web frontend for multiple architectures -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-web -f make/web/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-web -f make/web/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-web -f deploy/images/dockerfiles/web/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-web -f deploy/images/dockerfiles/web/Dockerfile . --push # 📚 build documentation for multiple architectures -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-docs -f make/docs/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-docs -f make/docs/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-docs -f deploy/images/dockerfiles/docs/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-docs -f deploy/images/dockerfiles/docs/Dockerfile . --push # 🔗 build MCP Server for multiple architectures -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-mcp -f make/mcp/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp -f make/mcp/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-mcp -f deploy/images/dockerfiles/mcp/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp -f deploy/images/dockerfiles/mcp/Dockerfile . --push # 💻 build Ubuntu Terminal for multiple architectures -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-terminal -f make/terminal/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-terminal -f make/terminal/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-terminal -f deploy/images/dockerfiles/terminal/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-terminal -f deploy/images/dockerfiles/terminal/Dockerfile . --push ``` ### 💻 Local Development Build ```bash # 🚀 Build application image (current architecture only) -docker build --progress=plain -t nexent/nexent -f make/main/Dockerfile . +docker build --progress=plain -t nexent/nexent -f deploy/images/dockerfiles/main/Dockerfile . # 📊 Build data process image (current architecture only) -docker build --progress=plain -t nexent/nexent-data-process -f make/data_process/Dockerfile . +docker build --progress=plain -t nexent/nexent-data-process -f deploy/images/dockerfiles/data-process/Dockerfile . + +# 📊 Build GPU data process image (current architecture only) +docker build --progress=plain -t nexent/nexent-data-process-gpu -f deploy/images/dockerfiles/data-process/Dockerfile --build-arg DATA_PROCESS_DEPENDENCY_VARIANT=gpu . # 🌐 Build web frontend image (current architecture only) -docker build --progress=plain -t nexent/nexent-web -f make/web/Dockerfile . +docker build --progress=plain -t nexent/nexent-web -f deploy/images/dockerfiles/web/Dockerfile . # 📚 Build documentation image (current architecture only) -docker build --progress=plain -t nexent/nexent-docs -f make/docs/Dockerfile . +docker build --progress=plain -t nexent/nexent-docs -f deploy/images/dockerfiles/docs/Dockerfile . # 🔗 Build MCP Server image (current architecture only) -docker build --progress=plain -t nexent/nexent-mcp -f make/mcp/Dockerfile . +docker build --progress=plain -t nexent/nexent-mcp -f deploy/images/dockerfiles/mcp/Dockerfile . # 💻 Build OpenSSH Server image (current architecture only) -docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f make/terminal/Dockerfile . +docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f deploy/images/dockerfiles/terminal/Dockerfile . + +# 💻 Build OpenSSH Server image with Conda (current architecture only) +docker build --progress=plain -t nexent/nexent-ubuntu-terminal-conda -f deploy/images/dockerfiles/terminal/Dockerfile --build-arg TERMINAL_VARIANT=conda . ``` ### 🧹 Clean up Docker resources @@ -62,52 +112,48 @@ docker builder prune -f && docker system prune -f #### Main Application Image (nexent/nexent) - Contains backend API service -- Built from `make/main/Dockerfile` +- Built from `deploy/images/dockerfiles/main/Dockerfile` - Provides core agent services #### Data Processing Image (nexent/nexent-data-process) - Contains data processing service -- Built from `make/data_process/Dockerfile` +- Built from `deploy/images/dockerfiles/data-process/Dockerfile` - Handles document parsing and vectorization #### Web Frontend Image (nexent/nexent-web) - Contains Next.js frontend application -- Built from `make/web/Dockerfile` +- Built from `deploy/images/dockerfiles/web/Dockerfile` - Provides user interface #### Documentation Image (nexent/nexent-docs) - Contains Vitepress documentation site -- Built from `make/docs/Dockerfile` +- Built from `deploy/images/dockerfiles/docs/Dockerfile` - Provides project documentation and API reference #### MCP Server Image (nexent/nexent-mcp) - Contains MCP (Model Context Protocol) proxy service -- Built from `make/mcp/Dockerfile` +- Built from `deploy/images/dockerfiles/mcp/Dockerfile` - Provides MCP server functionality for AI model integration ##### Pre-installed Tools and Features -- **Python Environment**: Python 3.10 + pip +- **Python Environment**: Python 3.11 + pip - **MCP Proxy**: mcp-proxy package for protocol handling - **Node.js**: Node.js 20.17.0 with npm - **Architecture Support**: linux/amd64, linux/arm64 -- **Base Image**: python:3.10-slim +- **Base Image**: python:3.11-slim #### OpenSSH Server Image (nexent/nexent-ubuntu-terminal) - Ubuntu 24.04-based SSH server container -- Built from `make/terminal/Dockerfile` -- Pre-installed with Conda, Python, Git and other development tools -- Supports SSH key authentication with username `linuxserver.io` -- Provides complete development environment +- Built from `deploy/images/dockerfiles/terminal/Dockerfile` +- Defaults to OpenSSH, Python, pip, venv, Git, Curl, and Wget +- `TERMINAL_VARIANT=conda` also installs Miniconda, Vim, and the compiler toolchain +- Runs as root and allows root login with password authentication ##### Pre-installed Tools and Features -- **Python Environment**: Python 3 + pip + virtualenv -- **Conda Management**: Miniconda3 environment management -- **Development Tools**: Git, Vim, Nano, Curl, Wget -- **Build Tools**: build-essential, Make -- **SSH Service**: Port 2222, root login and password authentication disabled -- **User Permissions**: `linuxserver.io` user has sudo privileges (no password required) -- **Timezone Setting**: Asia/Shanghai -- **Security Configuration**: SSH key authentication, 60-minute session timeout +- **Python Environment**: Python 3 + pip + venv +- **Conda Management**: Miniconda3 is included only in the `conda` variant +- **Development Tools**: Git, Curl, Wget; the `conda` variant also includes Vim and build-essential +- **SSH Service**: Container port 22, root login and password authentication enabled ### 🏷️ Tagging Strategy @@ -130,7 +176,7 @@ The documentation image can be built and run independently to serve nexent.tech/ ### Build Documentation Image ```bash -docker build -t nexent/nexent-docs -f make/docs/Dockerfile . +docker build -t nexent/nexent-docs -f deploy/images/dockerfiles/docs/Dockerfile . ``` ### Run Documentation Container @@ -178,11 +224,35 @@ Notes: ## 🚀 Deployment Recommendations -After building is complete, you can deploy local images from the `docker` directory: +After building is complete, you can deploy local images from the repository root: + +```bash +bash deploy.sh docker --image-source local-latest +``` + +> `local-latest` uses local `latest` Nexent application images and avoids pulling those images again. You do not need to modify `deploy/docker/deploy.sh`. + +### Package Local Images for Offline Deployment + +After building local `latest` images, package them with the offline builder: ```bash -cd docker -bash deploy.sh --image-source local-latest +bash deploy/offline/build_offline_package.sh \ + --target docker \ + --version latest \ + --platform amd64 \ + --components infrastructure,application,data-process,supabase \ + --image-source local-latest \ + --compress true \ + --output-dir offline-package/docker-local ``` -> `local-latest` uses local `latest` Nexent application images and avoids pulling those images again. You do not need to modify `docker/deploy.sh`. +When `--version latest` or `--image-source local-latest` is used, the builder expects local Nexent application images and skips pulling those `latest` tags. The package can then be moved to another host and deployed with: + +```bash +cd offline-package/docker-local +bash deploy.sh --load-images docker \ + --version latest \ + --components infrastructure,application,data-process,supabase \ + --image-source local-latest +``` diff --git a/doc/docs/en/developer-guide/environment-setup.md b/doc/docs/en/developer-guide/environment-setup.md index e2b0b9ed3..ec72dfdeb 100644 --- a/doc/docs/en/developer-guide/environment-setup.md +++ b/doc/docs/en/developer-guide/environment-setup.md @@ -21,9 +21,8 @@ Use this guide to prepare your environment before developing with Nexent. It sep Before backend work, start core services (PostgreSQL, Redis, Elasticsearch, MinIO, etc.). ```bash -# Run from the docker directory at the project root -cd docker -./deploy.sh --components infrastructure --port-policy development +# Run from the repository root +bash deploy.sh docker --components infrastructure --port-policy development ``` :::: info Important Notes diff --git a/doc/docs/en/quick-start/installation.md b/doc/docs/en/quick-start/installation.md index 7b6a9cb76..5c826cb4a 100644 --- a/doc/docs/en/quick-start/installation.md +++ b/doc/docs/en/quick-start/installation.md @@ -18,17 +18,17 @@ ```bash git clone https://github.com/ModelEngine-Group/nexent.git -cd nexent/docker +cd nexent ``` -> **💡 Tip**: `deploy.sh` automatically copies `.env.example` to `docker/.env` when `docker/.env` does not exist. If you need to configure voice models (STT/TTS), update the related values in `docker/.env` before or after deployment. +> **Tip**: Docker and Kubernetes use the project root `.env`. Existing `.env` is kept as-is. If it does not exist, the deploy scripts first reuse an existing `docker/.env`, then fall back to `.env.example` or `docker/.env.example`. If you need to configure voice models (STT/TTS), update the related values in `.env` before or after deployment. ### 2. Deployment Options Run the following command to start deployment: ```bash -bash deploy.sh +bash deploy.sh docker ``` After running the command, the script opens Bash TUI menus for deployment options. Use arrow keys or `j/k` to move, Space to toggle multi-select items, Enter to confirm, `b`/Backspace to go back, and `q` to quit. @@ -36,8 +36,8 @@ After running the command, the script opens Bash TUI menus for deployment option **Deployment Components:** - **infrastructure (required)**: Elasticsearch, PostgreSQL, Redis, MinIO - **application (selected by default, optional)**: config, runtime, mcp, northbound, web -- **data-process (optional)**: data processing service -- **supabase (optional)**: enables user, tenant, and authentication features +- **data-process (selected by default, optional)**: data processing service +- **supabase (selected by default, optional)**: enables user, tenant, and authentication features - **terminal (optional)**: enables the OpenSSH terminal tool - **monitoring (optional)**: enables observability components and then prompts for a provider @@ -54,19 +54,19 @@ You can also pass options directly: ```bash # Default component set, development port policy, standard image source -bash deploy.sh --components infrastructure,application --port-policy development --image-source general +bash deploy.sh docker --components infrastructure,application,data-process,supabase --port-policy development --image-source general -# Enable user/tenant features, data processing, and terminal -bash deploy.sh --components infrastructure,application,supabase,data-process,terminal +# Add the terminal tool to the default component set +bash deploy.sh docker --components infrastructure,application,data-process,supabase,terminal # Use mainland China image sources -bash deploy.sh --image-source mainland +bash deploy.sh docker --image-source mainland # Use local latest images -bash deploy.sh --image-source local-latest +bash deploy.sh docker --image-source local-latest ``` -After a successful deployment, non-sensitive choices are saved to `docker/deploy.options`. The next interactive deployment can reuse the local config or run a full reconfiguration. +After a successful deployment, non-sensitive choices are saved to `deploy/docker/deploy.options`. The next interactive deployment can reuse the local config or run a full reconfiguration. #### ⚠️ Important Notes @@ -152,7 +152,52 @@ Nexent uses Docker volumes for data persistence: Default `dataDir` is `./volumes` (configurable via `ROOT_DIR` in `.env`). -Uninstall is handled by `docker/uninstall.sh`. It prompts before deleting persistent data by default; you can also pass `--delete-volumes true|false`, `--remove-volumes`, `--keep-volumes`, or use `bash uninstall.sh delete-all` to remove containers and persistent data. +### Uninstall Docker Deployment + +Use the root uninstall entrypoint from the repository root: + +```bash +# Stop and remove containers; keep persistent data unless you confirm deletion +bash uninstall.sh docker + +# Non-interactive uninstall that keeps data +bash uninstall.sh docker --keep-volumes + +# Delete Docker volumes and Nexent data under ROOT_DIR +bash uninstall.sh docker --delete-volumes true + +# Full cleanup: containers plus persistent data +bash uninstall.sh docker delete-all +``` + +The Docker uninstall script reads `.env` to resolve `ROOT_DIR` and removes Compose resources. Data deletion removes service directories such as `postgresql`, `elasticsearch`, `redis`, `minio`, `volumes`, `openssh-server`, `scripts`, and `skills`; keep volumes when you plan to redeploy with existing data. + +### Offline Image Package + +Use `deploy/offline/build_offline_package.sh` when you need to move images and deployment scripts to an offline host: + +```bash +bash deploy/offline/build_offline_package.sh \ + --target docker \ + --version v2.2.1 \ + --platform amd64 \ + --components infrastructure,application,data-process,supabase \ + --image-source general \ + --compress true \ + --output-dir offline-package/docker +``` + +The package directory contains `images/*.tar`, `load-images.sh`, `deploy.sh`, `uninstall.sh`, `manifest.yaml`, `checksums.txt`, `.env.example`, and `deploy/sql`. It does not include local `.env` or `deploy.options`. With `--compress true`, a `nexent-offline---.zip` archive is created next to the output directory. + +On the target host, keep the deployment options consistent with the package manifest: + +```bash +cd offline-package/docker +bash deploy.sh --load-images docker \ + --version v2.2.1 \ + --components infrastructure,application,data-process,supabase \ + --image-source general +``` ## 🔌 Port Mapping @@ -175,14 +220,14 @@ For complete port mapping details, see our [Dev Container Guide](../deployment/d ### Monitoring Configuration -Select the `monitoring` component in the deployment script UI to enable OpenTelemetry monitoring. The script synchronizes `ENABLE_TELEMETRY`, `MONITORING_PROVIDER`, and `MONITORING_DASHBOARD_URL` in `docker/.env`, then starts the matching observability services from `docker/docker-compose-monitoring.yml`. +Select the `monitoring` component in the deployment script UI to enable OpenTelemetry monitoring. The script synchronizes `ENABLE_TELEMETRY`, `MONITORING_PROVIDER`, and `MONITORING_DASHBOARD_URL` in `.env`, then starts the matching observability services from `deploy/docker/compose/docker-compose-monitoring.yml`. ```bash -cd nexent/docker -bash deploy.sh +cd nexent +bash deploy.sh docker ``` -If `docker/deploy.options` already exists, the script asks whether to reuse local configuration. Choose to reconfigure/overwrite local configuration, then select `monitoring` in the component menu and manually choose `grafana`, `phoenix`, `langfuse`, `langsmith`, `zipkin`, or `otlp` in the provider menu. +If `deploy/docker/deploy.options` already exists, the script asks whether to reuse local configuration. Choose to reconfigure/overwrite local configuration, then select `monitoring` in the component menu and manually choose `grafana`, `phoenix`, `langfuse`, `langsmith`, `zipkin`, or `otlp` in the provider menu. Supported providers: @@ -198,7 +243,7 @@ Supported providers: To change ports, image versions, or local Langfuse bootstrap credentials, copy and edit the monitoring environment file first: ```bash -cp docker/monitoring/monitoring.env.example docker/monitoring/monitoring.env +cp deploy/docker/assets/monitoring/monitoring.env.example deploy/docker/assets/monitoring/monitoring.env ``` Common variables: @@ -211,7 +256,7 @@ Common variables: | `LANGFUSE_INIT_USER_EMAIL` / `LANGFUSE_INIT_USER_PASSWORD` | Local Langfuse bootstrap admin | | `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD` | Local Grafana admin | -Before choosing the `langsmith` provider, configure `LANGSMITH_API_KEY` in `docker/monitoring/monitoring.env`. If you only need to connect to an existing external Collector, adjust the OTLP target in `docker/.env`: +Before choosing the `langsmith` provider, configure `LANGSMITH_API_KEY` in `deploy/docker/assets/monitoring/monitoring.env`. If you only need to connect to an existing external Collector, adjust the OTLP target in `.env`: ```bash ENABLE_TELEMETRY=true @@ -228,10 +273,10 @@ MONITORING_DASHBOARD_URL= OAuth login requires the `supabase` component. When enabling third-party login, deploy `supabase` and set `OAUTH_CALLBACK_BASE_URL` to the browser-accessible Nexent Web URL. ```bash -bash deploy.sh --components infrastructure,application,supabase +bash deploy.sh docker --components infrastructure,application,supabase ``` -For Docker, configure OAuth in `docker/.env`: +For Docker, configure OAuth in `.env`: ```bash # Web entry URL. The full callback path is generated as: @@ -277,7 +322,7 @@ For local Docker, a GitHub callback example is `http://localhost:3000/api/user/o CAS SSO does not require the `supabase` component. Set `CAS_CALLBACK_BASE_URL` to the browser-accessible Nexent Web URL without a trailing `/`. `CAS_SERVER_URL` is the CAS Server root URL and should also not include a trailing `/`. -For Docker, configure CAS in `docker/.env`: +For Docker, configure CAS in `.env`: ```bash CAS_ENABLED=true diff --git a/doc/docs/en/quick-start/kubernetes-installation.md b/doc/docs/en/quick-start/kubernetes-installation.md index a10873c7c..d5eb828b4 100644 --- a/doc/docs/en/quick-start/kubernetes-installation.md +++ b/doc/docs/en/quick-start/kubernetes-installation.md @@ -27,7 +27,7 @@ kubectl get nodes ```bash git clone https://github.com/ModelEngine-Group/nexent.git -cd nexent/k8s/helm +cd nexent ``` ### 3. Deployment @@ -35,7 +35,7 @@ cd nexent/k8s/helm Run the deployment script: ```bash -./deploy.sh +bash deploy.sh k8s ``` After running the command, the script opens Bash TUI menus for configuration. Use arrow keys or `j/k` to move, Space to toggle multi-select items, Enter to confirm, `b`/Backspace to go back, and `q` to quit. @@ -43,8 +43,8 @@ After running the command, the script opens Bash TUI menus for configuration. Us **Deployment Components:** - **infrastructure (required)**: Elasticsearch, PostgreSQL, Redis, MinIO - **application (selected by default, optional)**: config, runtime, mcp, northbound, web -- **data-process (optional)**: data processing service -- **supabase (optional)**: enables user, tenant, and authentication features +- **data-process (selected by default, optional)**: data processing service +- **supabase (selected by default, optional)**: enables user, tenant, and authentication features - **terminal (optional)**: enables the OpenSSH terminal tool - **monitoring (optional)**: enables observability components and then prompts for a provider @@ -57,7 +57,9 @@ After running the command, the script opens Bash TUI menus for configuration. Us - **mainland**: uses mainland China mirrors - **local-latest**: uses local `latest` images and local-friendly pull policies for Nexent application images -After a successful deployment, non-sensitive choices are saved to `k8s/helm/deploy.options`. The next interactive deployment can reuse the local config or run a full reconfiguration. +Kubernetes uses the same project root `.env` as Docker. Existing `.env` is kept as-is. If it does not exist, the deploy scripts first reuse an existing `docker/.env`, then fall back to `.env.example` or `docker/.env.example`. + +After a successful deployment, non-sensitive choices are saved to `deploy/k8s/deploy.options`. The next interactive deployment can reuse the local config or run a full reconfiguration. ### ⚠️ Important Notes @@ -80,7 +82,7 @@ kubectl exec -it -n nexent deploy/nexent-postgresql -- psql -U root -d nexent -c "DELETE FROM nexent.user_tenant_t WHERE user_id='your_user_id';" # Step 3: Re-deploy and record the su account password -./deploy.sh +bash deploy.sh k8s ``` ### 4. Access Your Installation @@ -155,44 +157,96 @@ Nexent uses PersistentVolumes for data persistence: | Redis | nexent-redis-pv | `/var/lib/nexent-data/nexent-redis` | | MinIO | nexent-minio-pv | `/var/lib/nexent-data/nexent-minio` | | Supabase DB (when `supabase` is selected) | nexent-supabase-db-pv | `/var/lib/nexent-data/nexent-supabase-db` | +| Shared workspace | nexent-workspace-pv | `/var/lib/nexent` | +| Shared skills | nexent-skills-pv | `/var/lib/nexent-data/skills` | + +Helm uninstall does not delete local hostPath data by default. Use `bash deploy/k8s/uninstall.sh --delete-local-data true` or `bash uninstall.sh k8s --delete-local-data true` to delete known Nexent local volume contents under `/var/lib/nexent`, `/var/lib/nexent-data/skills`, and `/var/lib/nexent-data/nexent-*`; use `--keep-local-data` to preserve them explicitly. + +### Uninstall Kubernetes Deployment + +Use the root uninstall entrypoint from the repository root: + +```bash +# Remove Helm release; prompts before deleting namespace or local data in interactive shells +bash uninstall.sh k8s + +# Clean only Helm release state, useful for stuck releases +bash uninstall.sh k8s clean + +# Remove Helm release and namespace, but keep local hostPath data +bash uninstall.sh k8s delete --keep-local-data + +# Delete known local hostPath data after uninstall +bash uninstall.sh k8s --delete-local-data true + +# Full cleanup: Helm release, namespace, and local hostPath data +bash uninstall.sh k8s delete-all +``` + +`--delete-data` and `--delete-volumes` are compatibility options for Helm-managed resources. For local disks, use `--delete-local-data` or `--keep-local-data`; `delete-all --keep-local-data` removes the namespace while preserving local volume contents. + +### Offline Image Package + +Build a Kubernetes offline package from the repository root: + +```bash +bash deploy/offline/build_offline_package.sh \ + --target k8s \ + --version v2.2.1 \ + --platform amd64 \ + --components infrastructure,application,data-process,supabase \ + --image-source general \ + --compress true \ + --output-dir offline-package/k8s +``` + +The package includes image tar files, `load-images.sh`, root deploy/uninstall entrypoints, Kubernetes Helm assets, SQL files, `manifest.yaml`, and `checksums.txt`. With `--compress true`, a `nexent-offline---.zip` archive is created next to the output directory. On a single-node Docker-backed cluster, you can load and deploy directly: + +```bash +cd offline-package/k8s +bash deploy.sh --load-images k8s \ + --version v2.2.1 \ + --components infrastructure,application,data-process,supabase \ + --image-source general +``` -Helm uninstall does not delete local hostPath data by default. Use `./uninstall.sh --delete-local-data true` to delete known Nexent local volume contents under `/var/lib/nexent-data/nexent-*`, or `--keep-local-data` to preserve them explicitly. +For multi-node clusters, load the images on every node that may run Nexent Pods, or push the loaded images to an internal registry and deploy with matching image settings. ## 🔧 Deployment Commands ```bash # Deploy with interactive prompts -./deploy.sh +bash deploy.sh k8s # Non-interactive deployment with the default component set -./deploy.sh --components infrastructure,application --port-policy development --image-source general +bash deploy.sh k8s --components infrastructure,application,data-process,supabase --port-policy development --image-source general -# Enable user/tenant features, data processing, and terminal -./deploy.sh --components infrastructure,application,supabase,data-process,terminal +# Add the terminal tool to the default component set +bash deploy.sh k8s --components infrastructure,application,data-process,supabase,terminal # Deploy with mainland China image sources -./deploy.sh --image-source mainland +bash deploy.sh k8s --image-source mainland # Use local latest images -./deploy.sh --image-source local-latest +bash deploy.sh k8s --image-source local-latest # Clean helm state only (fixes stuck releases) -./uninstall.sh clean +bash uninstall.sh k8s clean # Uninstall; local data is preserved by default, with interactive prompts for namespace and local data deletion -./uninstall.sh +bash uninstall.sh k8s # Uninstall and delete the namespace -./uninstall.sh --delete-namespace true +bash uninstall.sh k8s --delete-namespace true # Uninstall and delete local hostPath data -./uninstall.sh --delete-local-data true +bash uninstall.sh k8s --delete-local-data true # Complete uninstall including namespace and local hostPath data -./uninstall.sh delete-all +bash uninstall.sh k8s delete-all # Complete uninstall but preserve local hostPath data -./uninstall.sh delete-all --keep-local-data +bash uninstall.sh k8s delete-all --keep-local-data ``` ## 🔧 Advanced Configuration @@ -202,11 +256,11 @@ Helm uninstall does not delete local hostPath data by default. Use `./uninstall. Kubernetes deployments enable monitoring through the `monitoring` component in the deployment script UI. The deployment script renders runtime Helm values for `global.monitoring.enabled`, `global.monitoring.provider`, and `global.monitoring.dashboardUrl`, and enables the `nexent-monitoring` subchart. ```bash -cd nexent/k8s/helm -./deploy.sh +cd nexent +bash deploy.sh k8s ``` -If `k8s/helm/deploy.options` already exists, the script asks whether to reuse local configuration. Choose to reconfigure/overwrite local configuration, then select `monitoring` in the component menu and manually choose `grafana`, `phoenix`, `langfuse`, `langsmith`, `zipkin`, or `otlp` in the provider menu. +If `deploy/k8s/deploy.options` already exists, the script asks whether to reuse local configuration. Choose to reconfigure/overwrite local configuration, then select `monitoring` in the component menu and manually choose `grafana`, `phoenix`, `langfuse`, `langsmith`, `zipkin`, or `otlp` in the provider menu. Supported providers: @@ -219,7 +273,7 @@ Supported providers: | `grafana` | Local Grafana + Tempo | `http://localhost:30002/d/nexent-llm-agent/nexent-agent-trace-monitoring?orgId=1` | | `zipkin` | Local Zipkin | `http://localhost:30011` | -Before choosing the `langsmith` provider, configure `global.monitoring.langsmithApiKey` and `global.monitoring.langsmithProject` in `k8s/helm/nexent/values.yaml`. To change local Grafana, Langfuse, or dashboard ports, adjust the values file first, then re-run the deployment script, choose to reconfigure, and manually select `monitoring`. +Before choosing the `langsmith` provider, configure `global.monitoring.langsmithApiKey` and `global.monitoring.langsmithProject` in `deploy/deploy/k8s/helm/nexent/values.yaml`. To change local Grafana, Langfuse, or dashboard ports, adjust the values file first, then re-run the deployment script, choose to reconfigure, and manually select `monitoring`. Common Helm values: @@ -248,7 +302,7 @@ kubectl get svc -n nexent | grep -E 'otel|phoenix|grafana|zipkin|langfuse' OAuth login requires the `supabase` component. When enabling third-party login, deploy `supabase` and set `config.oauth.callbackBaseUrl` to the browser-accessible Nexent Web URL. ```bash -./deploy.sh --components infrastructure,application,supabase +bash deploy.sh k8s --components infrastructure,application,supabase ``` Kubernetes writes OAuth settings into backend environment variables through `nexent-common` `config.oauth.*` values: diff --git a/doc/docs/en/quick-start/kubernetes-upgrade-guide.md b/doc/docs/en/quick-start/kubernetes-upgrade-guide.md index 75afcfba9..83850aa40 100644 --- a/doc/docs/en/quick-start/kubernetes-upgrade-guide.md +++ b/doc/docs/en/quick-start/kubernetes-upgrade-guide.md @@ -14,7 +14,7 @@ Follow these steps to upgrade Nexent on Kubernetes safely: Before updating, record the current deployment version and data directory information. -- Current Deployment Version Location: `APP_VERSION` in `backend/consts/const.py` +- Current Deployment Version Location: root `VERSION` - Local volume directories: each Helm sub-chart's `storage.hostPath`, defaulting to `/var/lib/nexent-data/nexent-*` **Code downloaded via git** @@ -28,15 +28,14 @@ git pull **Code downloaded via ZIP package or other means** 1. Re-download the latest code from GitHub and extract it. -2. Copy the `deploy.options` file from the `k8s/helm` directory of your previous deployment to the new code directory. (If the file doesn't exist, you can ignore this step). +2. Copy the `deploy.options` file from the `deploy/k8s` directory of your previous deployment to the same directory in the new code. (If the file does not exist, you can ignore this step). ## 🔄 Step 2: Execute the Upgrade -Navigate to the k8s/helm directory of the updated code and run the deployment script: +From the repository root of the updated code, run the Kubernetes deployment entrypoint: ```bash -cd k8s/helm -./deploy.sh +bash deploy.sh k8s ``` The script will detect your saved deployment settings (components, port policy, image source, etc.) from `deploy.options`. If the file is missing, you will be prompted to enter configuration details. @@ -55,79 +54,11 @@ After deployment: --- -## 🗄️ Manual Database Update +## 🗄️ Database Migrations -If some SQL files fail to execute during the upgrade, or if you need to run incremental SQL scripts manually, you can perform the update using the methods below. +SQL migrations are no longer executed manually. In Kubernetes, only `nexent-config` runs `deploy/common/run-sql-migrations.sh` on startup and automatically applies `*.sql` files from `deploy/sql/migrations/` in filename order; the other backend services only wait for migration records to reach the target state. The deploy script renders `deploy/sql` into the shared SQL ConfigMap mounted at `/opt/nexent/sql`, so SQL-only changes require rerunning deployment, not rebuilding images. -### 📋 Find SQL Scripts - -SQL migration scripts are located in the repository at: - -``` -docker/sql/ -``` - -Check the [upgrade-guide](./upgrade-guide.md) or release notes to identify which SQL scripts need to be executed for your upgrade path. - -### ✅ Method A: Use a SQL Editor (recommended) - -1. Open your SQL client and create a new PostgreSQL connection. -2. Get connection settings from the running PostgreSQL pod: - - ```bash - # Get PostgreSQL pod name - kubectl get pods -n nexent -l app=nexent-postgresql - - # Port-forward to access PostgreSQL locally - kubectl port-forward svc/nexent-postgresql 5433:5432 -n nexent & - ``` - -3. Connection details: - - Host: `localhost` - - Port: `5433` (forwarded port) - - Database: `nexent` - - User: `root` - - Password: Check in `k8s/helm/nexent/charts/nexent-common/values.yaml` - -4. Test the connection. When successful, you should see tables under the `nexent` schema. -5. Execute the required SQL file(s) in version order. - -> ⚠️ Important -> - Always back up the database first, especially in production. -> - Run scripts sequentially to avoid dependency issues. - -### 🧰 Method B: Use kubectl exec (no SQL client required) - -Execute SQL scripts directly via stdin redirection: - -1. Get the PostgreSQL pod name: - - ```bash - kubectl get pods -n nexent -l app=nexent-postgresql -o jsonpath='{.items[0].metadata.name}' - ``` - -2. Execute the SQL file directly from your host machine: - - ```bash - kubectl exec -i -n nexent -- psql -U root -d nexent < ./sql/v1.1.1_1030-update.sql - ``` - - Or if you want to see the output interactively: - - ```bash - cat ./sql/v1.1.1_1030-update.sql | kubectl exec -i -n nexent -- psql -U root -d nexent - ``` - -**Example - Execute multiple SQL files:** - -```bash -# Get PostgreSQL pod name -POSTGRES_POD=$(kubectl get pods -n nexent -l app=nexent-postgresql -o jsonpath='{.items[0].metadata.name}') - -# Execute SQL files in order -kubectl exec -i $POSTGRES_POD -n nexent -- psql -U root -d nexent < ./sql/v1.8.0_xxxxx-update.sql -kubectl exec -i $POSTGRES_POD -n nexent -- psql -U root -d nexent < ./sql/v2.0.0_0314_add_context_skill_t.sql -``` +The migration runner uses each SQL filename as the migration ID in `nexent.schema_migrations`. If a recorded file has the same checksum, it is skipped; if the checksum changes, the same file is rerun and the checksum, execution time, app version, and source file are updated. > 💡 Tips > - Create a backup before running migrations: @@ -137,13 +68,7 @@ kubectl exec -i $POSTGRES_POD -n nexent -- psql -U root -d nexent < ./sql/v2.0.0 kubectl exec nexent/$POSTGRES_POD -n nexent -- pg_dump -U root nexent > backup_$(date +%F).sql ``` -> - For the Supabase database (when `supabase` is selected), use the `nexent-supabase-db` pod instead: - - ```bash - SUPABASE_POD=$(kubectl get pods -n nexent -l app=nexent-supabase-db -o jsonpath='{.items[0].metadata.name}') - kubectl cp docker/sql/xxx.sql nexent/$SUPABASE_POD:/tmp/update.sql - kubectl exec -it nexent/$SUPABASE_POD -n nexent -- psql -U postgres -f /tmp/update.sql - ``` +> - Supabase initialization SQL is rendered from `deploy/sql/supabase/` into Helm values by the deploy script. It does not need to be copied or executed manually. --- @@ -163,9 +88,7 @@ kubectl logs -n nexent -l app=nexent-config --tail=100 kubectl logs -n nexent -l app=nexent-web --tail=100 ``` -### Restart Services After Manual SQL Update（if needed） - -If you executed SQL scripts manually, restart the affected services: +### Restart Services After Migration Retry ```bash kubectl rollout restart deployment/nexent-config -n nexent @@ -175,6 +98,5 @@ kubectl rollout restart deployment/nexent-runtime -n nexent ### Re-initialize Elasticsearch (if needed) ```bash -cd k8s/helm -bash init-elasticsearch.sh +bash deploy/k8s/init-elasticsearch.sh ``` diff --git a/doc/docs/en/quick-start/upgrade-guide.md b/doc/docs/en/quick-start/upgrade-guide.md index 3bc22f254..be8882506 100644 --- a/doc/docs/en/quick-start/upgrade-guide.md +++ b/doc/docs/en/quick-start/upgrade-guide.md @@ -14,8 +14,8 @@ Follow these steps to upgrade Nexent safely: Before updating, record the current deployment version and data directory information. -- Current Deployment Version Location: APP_VERSION in backend/consts/const.py -- Data Directory Location: ROOT_DIR in docker/.env +- Current Deployment Version Location: root VERSION +- Data Directory Location: ROOT_DIR in .env **Code downloaded via git** @@ -32,17 +32,17 @@ git pull ## 🔄 Step 2: Execute the Upgrade -Navigate to the docker directory of the updated code and run the upgrade script: +From the repository root of the updated code, run the Docker deployment entrypoint: ```bash -bash upgrade.sh +bash deploy.sh docker ``` If deploy.options is missing, the script will prompt you to select deployment settings again, such as components, port policy, and image source. Choose the same options you used for the previous deployment. >💡 Tip -> If `docker/.env` is missing, the deploy script automatically copies it from `.env.example`. -> If you need to configure voice models (STT/TTS), add the relevant variables to `docker/.env`. We will provide a front-end configuration interface as soon as possible. +> Existing `.env` is kept as-is. If it is missing, the deploy script first reuses an existing `docker/.env`, then falls back to `.env.example` or `docker/.env.example`. +> If you need to configure voice models (STT/TTS), add the relevant variables to `.env`. We will provide a front-end configuration interface as soon as possible. ## 🌐 Step 3: Verify the deployment @@ -82,74 +82,12 @@ docker system prune -af --- -## 🗄️ Manual Database Update +## 🗄️ Database Migrations -If some SQL files fail to execute during the upgrade, you can perform the update manually. +SQL migrations are no longer executed manually. In Docker, only `nexent-config` runs `deploy/common/run-sql-migrations.sh` on startup and automatically applies `*.sql` files from `deploy/sql/migrations/` in filename order; the other backend containers only wait for migration records to reach the target state. SQL is mounted from `deploy/sql` into `/opt/nexent/sql`, so SQL-only changes require rerunning deployment, not rebuilding images. -### ✅ Method A: Use a SQL editor (recommended) - -1. Open your SQL client and create a new PostgreSQL connection. -2. Retrieve connection settings from `/nexent/docker/.env`: - - Host - - Port - - Database - - User - - Password -3. Test the connection. When successful, you should see tables under the `nexent` schema. -4. Open a new query window. -5. Navigate to the /nexent/docker/sql directory and open the failed SQL file(s) to view the script. -6. Execute the failed SQL file(s) and any subsequent version SQL files in order. - -> ⚠️ Important -> - Always back up the database first, especially in production. -> - Run scripts sequentially to avoid dependency issues. -> - `.env` keys may be named `POSTGRES_HOST`, `POSTGRES_PORT`, and so on—map them accordingly in your SQL client. - -### 🧰 Method B: Use the command line (no SQL client required) - -1. Switch to the Docker directory: - - ```bash - cd nexent/docker - ``` - -2. Read database connection details from `.env`, for example: - - ```bash - POSTGRES_HOST=localhost - POSTGRES_PORT=5432 - POSTGRES_DB=nexent - POSTGRES_USER=root - POSTGRES_PASSWORD=your_password - ``` - -3. Execute SQL files sequentially (host machine example): - - ```bash - # execute the following commands (please replace the placeholders with your actual values) - docker exec -i nexent-postgresql psql -U [YOUR_POSTGRES_USER] -d [YOUR_POSTGRES_DB] < ./sql/v1.1.1_1030-update.sql - docker exec -i nexent-postgresql psql -U [YOUR_POSTGRES_USER] -d [YOUR_POSTGRES_DB] < ./sql/v1.1.2_1105-update.sql - ``` - - Execute the corresponding scripts for your deployment versions in version order. +The migration runner uses each SQL filename as the migration ID in `nexent.schema_migrations`. If a recorded file has the same checksum, it is skipped; if the checksum changes, the same file is rerun and the checksum, execution time, app version, and source file are updated. > 💡 Tips -> - Load environment variables first if they are defined in `.env`: -> -> **Windows PowerShell:** -> ```powershell -> Get-Content .env | Where-Object { $_ -notmatch '^#' -and $_ -match '=' } | ForEach-Object { $key, $value = $_ -split '=', 2; [Environment]::SetEnvironmentVariable($key.Trim(), $value.Trim(), 'Process') } -> ``` -> -> **Linux/WSL:** -> ```bash -> export $(grep -v '^#' .env | xargs) -> # Or use set -a to automatically export all variables -> set -a; source .env; set +a -> ``` -> -> - Create a backup before running migrations: -> -> ```bash -> docker exec -i nexent-postgres pg_dump -U [YOUR_POSTGRES_USER] [YOUR_POSTGRES_DB] > backup_$(date +%F).sql -> ``` +> - Always back up the database before upgrading, especially in production. +> - Check backend container logs for `[sql-migrations]` entries if a service fails during startup. diff --git a/doc/docs/en/sdk/monitoring.md b/doc/docs/en/sdk/monitoring.md index bb7c1db13..2c90180c6 100644 --- a/doc/docs/en/sdk/monitoring.md +++ b/doc/docs/en/sdk/monitoring.md @@ -15,17 +15,17 @@ NexentAgent ──► OpenTelemetry SDK ──► OTLP Collector ──► Arize ## Quick Start ```bash -cd docker -[ -f .env ] || cp .env.example .env -cp monitoring/monitoring.env.example monitoring/monitoring.env +cd deploy/docker +[ -f ../../.env ] || cp ../../.env.example ../../.env +cp assets/monitoring/monitoring.env.example assets/monitoring/monitoring.env -vim .env +vim ../../.env ENABLE_TELEMETRY=true MONITORING_PROVIDER=otlp OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http -vim monitoring/monitoring.env +vim assets/monitoring/monitoring.env MONITORING_PROVIDER=otlp ./start-monitoring.sh --stack collector @@ -89,8 +89,8 @@ LangSmith supports online OTLP trace ingestion through the OpenTelemetry endpoin **Collector forwarding:** ```bash -cd docker -vim monitoring/monitoring.env +cd deploy/docker +vim assets/monitoring/monitoring.env MONITORING_PROVIDER=langsmith LANGSMITH_API_KEY=lsv2_xxx @@ -293,7 +293,7 @@ service: exporters: [otlphttp/langsmith, debug] ``` -See `docker/monitoring/otel-collector-config.yml` for full configuration with platform examples. +See `deploy/docker/assets/monitoring/otel-collector-config.yml` for full configuration with platform examples. ## Graceful Degradation diff --git a/doc/docs/en/user-guide/local-tools/terminal-tool.md b/doc/docs/en/user-guide/local-tools/terminal-tool.md index 45cfa67df..64f1b8289 100644 --- a/doc/docs/en/user-guide/local-tools/terminal-tool.md +++ b/doc/docs/en/user-guide/local-tools/terminal-tool.md @@ -33,7 +33,7 @@ Working directory: /opt/terminal ##### Method B: Local Image Build ```bash # Build Ubuntu Terminal image locally -docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f make/terminal/Dockerfile . +docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f deploy/images/dockerfiles/terminal/Dockerfile . ``` > 📚 **Detailed Build Instructions**: Refer to [Docker Build Guide](/en/deployment/docker-build) for complete image build and push processes. @@ -43,15 +43,12 @@ docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f make/terminal/ When running the deployment script, choose to enable the Terminal tool container: ```bash -# Run deployment script -cd docker -bash deploy.sh +# Run deployment script from the repository root +bash deploy.sh docker --components infrastructure,application,data-process,supabase,terminal # During script execution, select: -# 1. Deployment mode: Choose development/production/infrastructure mode -# 2. Terminal tool: Choose "Y" to enable Terminal tool container -# 3. Configure SSH credentials: Enter username and password -# 4. Configure mount directory: Specify host directory mapping +# During script execution, select or keep the terminal component enabled. +# Then configure SSH credentials and the host mount directory when prompted. ``` #### 3. Container Features diff --git a/doc/docs/zh/deployment/devcontainer.md b/doc/docs/zh/deployment/devcontainer.md index b5b934187..b22f0e490 100644 --- a/doc/docs/zh/deployment/devcontainer.md +++ b/doc/docs/zh/deployment/devcontainer.md @@ -25,8 +25,8 @@ 1. 克隆项目到本地 2. 在 Cursor 中打开项目文件夹 -3. 在 `docker` 目录运行 `./deploy.sh --components infrastructure,application --port-policy development` 启动基础容器 -4. 进入 `nexent-minio` 与 `nexent-elasticsearch` 容器, 将 `MINIO_ACCESS_KEY`, `MINIO_SECRET_KEY`, `ELASTICSEARCH_API_KEY` 环境变量复制到 `docker/docker-compose.dev.yml` 中的相应环境变量位置 +3. 在项目根目录运行 `bash deploy.sh docker --components infrastructure,application,data-process,supabase --port-policy development` 启动基础容器 +4. 进入 `nexent-minio` 与 `nexent-elasticsearch` 容器, 将 `MINIO_ACCESS_KEY`, `MINIO_SECRET_KEY`, `ELASTICSEARCH_API_KEY` 环境变量复制到 `deploy/docker/compose/docker-compose.dev.yml` 中的相应环境变量位置 5. 按下 `F1` 或 `Ctrl+Shift+P`，输入 `Dev Containers: Reopen in Container ...` 6. Cursor 将根据 `.devcontainer` 目录中的配置启动开发容器 @@ -54,7 +54,7 @@ 您可以通过修改以下文件来自定义开发环境： - `.devcontainer/devcontainer.json` - 插件配置项 -- `docker/docker-compose.dev.yml` - 开发容器的具体构筑项，需要修改环境变量值才能正常启动 +- `deploy/docker/compose/docker-compose.dev.yml` - 开发容器的具体构筑项，需要修改环境变量值才能正常启动 ## 6. 常见问题解决 @@ -68,4 +68,4 @@ sudo chown -R $(id -u):$(id -g) /opt 1. 重建容器：按下 `F1` 或 `Ctrl+Shift+P`，输入 `Dev Containers: Rebuild Container` 2. 检查 Docker 日志：`docker logs nexent-dev` -3. 检查 `.env` 文件中的配置是否正确 \ No newline at end of file +3. 检查 `.env` 文件中的配置是否正确 diff --git a/doc/docs/zh/deployment/docker-build.md b/doc/docs/zh/deployment/docker-build.md index 8e360d95d..a389aabd4 100644 --- a/doc/docs/zh/deployment/docker-build.md +++ b/doc/docs/zh/deployment/docker-build.md @@ -4,107 +4,153 @@ ## 🏗️ 构建和推送镜像 +推荐使用统一构建入口： + +```bash +# 类似部署脚本，进入交互式选择 +bash deploy/images/build.sh + +# 按镜像构建指定版本 +bash deploy/images/build.sh \ + --images main,web,mcp,data-process,terminal \ + --version v2.2.1 \ + --registry general \ + --platform linux/amd64,linux/arm64 \ + --push + +# 按同一镜像集合构建 latest 镜像 +bash deploy/images/build.sh \ + --images main,web,mcp,data-process \ + --version latest \ + --registry general \ + --platform linux/amd64 \ + --load + +# 需要时也可以只构建一个或多个指定镜像 +bash deploy/images/build.sh --web --docs --version v2.2.1 --dry-run +``` + +在终端无参数运行 `deploy/images/build.sh` 时，会依次选择镜像、镜像版本（`latest` 或根 `VERSION`）和镜像源。交互式默认选择 `main,web` 和 `latest`。也可以用 `--interactive` 强制进入同样的选择流程。 + +`--platform` 仅支持命令行传入。不传时不会添加 `--platform` 参数，默认按本地架构构建。 + +变体选项： +- `--dependency-variant cpu|gpu` 控制数据处理依赖，默认 `cpu`。`gpu` 会构建带 GPU/CUDA 依赖的镜像，并使用 `-gpu` 镜像名后缀。 +- `--terminal-variant slim|conda` 控制终端镜像，默认 `slim`。`conda` 会保留 Miniconda、`vim` 和编译工具链，并使用 `-conda` 镜像名后缀。 + +构建 `data-process` 时，`deploy/images/build.sh` 会自动准备 `model-assets`：优先使用仓库根目录已有的 `model-assets`，其次复用 `~/model-assets`，否则从 Hugging Face 仓库拉取并执行 `git lfs pull`。如果直接执行 `docker build`，需要先在仓库根目录准备好 `model-assets`。 + +镜像选项： +- `--main` 构建 `nexent` +- `--web` 构建 `nexent-web` +- `--data-process` 构建 `nexent-data-process` +- `--mcp` 构建 `nexent-mcp` +- `--terminal` 构建 `nexent-ubuntu-terminal` +- `--docs` 构建 `nexent-docs` + ```bash # 🛠️ 创建并使用支持多架构构建的新构建器实例 docker buildx create --name nexent_builder --use # 🚀 为多个架构构建应用程序 -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent -f make/main/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent -f make/web/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent -f deploy/images/dockerfiles/main/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent -f deploy/images/dockerfiles/web/Dockerfile . --push # 📊 为多个架构构建数据处理服务 -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-data-process -f make/data_process/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process -f make/web/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-data-process -f deploy/images/dockerfiles/data-process/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-data-process -f deploy/images/dockerfiles/web/Dockerfile . --push # 🌐 为多个架构构建前端 -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-web -f make/web/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-web -f make/web/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-web -f deploy/images/dockerfiles/web/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-web -f deploy/images/dockerfiles/web/Dockerfile . --push # 📚 为多个架构构建文档 -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-docs -f make/docs/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-docs -f make/docs/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-docs -f deploy/images/dockerfiles/docs/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-docs -f deploy/images/dockerfiles/docs/Dockerfile . --push # 🔗 为多个架构构建 MCP Server -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-mcp -f make/mcp/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp -f make/mcp/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-mcp -f deploy/images/dockerfiles/mcp/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-mcp -f deploy/images/dockerfiles/mcp/Dockerfile . --push # 💻 为多个架构构建 Ubuntu Terminal -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-terminal -f make/terminal/Dockerfile . --push -docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-terminal -f make/terminal/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t nexent/nexent-terminal -f deploy/images/dockerfiles/terminal/Dockerfile . --push +docker buildx build --progress=plain --platform linux/amd64,linux/arm64 -t ccr.ccs.tencentyun.com/nexent-hub/nexent-terminal -f deploy/images/dockerfiles/terminal/Dockerfile . --push ``` ## 💻 本地开发构建 ```bash # 🚀 构建应用程序镜像（仅当前架构） -docker build --progress=plain -t nexent/nexent -f make/main/Dockerfile . +docker build --progress=plain -t nexent/nexent -f deploy/images/dockerfiles/main/Dockerfile . # 📊 构建数据处理镜像（仅当前架构） -docker build --progress=plain -t nexent/nexent-data-process -f make/data_process/Dockerfile . +docker build --progress=plain -t nexent/nexent-data-process -f deploy/images/dockerfiles/data-process/Dockerfile . + +# 📊 构建 GPU 数据处理镜像（仅当前架构） +docker build --progress=plain -t nexent/nexent-data-process-gpu -f deploy/images/dockerfiles/data-process/Dockerfile --build-arg DATA_PROCESS_DEPENDENCY_VARIANT=gpu . # 🌐 构建前端镜像（仅当前架构） -docker build --progress=plain -t nexent/nexent-web -f make/web/Dockerfile . +docker build --progress=plain -t nexent/nexent-web -f deploy/images/dockerfiles/web/Dockerfile . # 📚 构建文档镜像（仅当前架构） -docker build --progress=plain -t nexent/nexent-docs -f make/docs/Dockerfile . +docker build --progress=plain -t nexent/nexent-docs -f deploy/images/dockerfiles/docs/Dockerfile . # 🔗 构建 MCP Server 镜像（仅当前架构） -docker build --progress=plain -t nexent/nexent-mcp -f make/mcp/Dockerfile . +docker build --progress=plain -t nexent/nexent-mcp -f deploy/images/dockerfiles/mcp/Dockerfile . # 💻 构建 OpenSSH Server 镜像（仅当前架构） -docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f make/terminal/Dockerfile . +docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f deploy/images/dockerfiles/terminal/Dockerfile . + +# 💻 构建带 Conda 的 OpenSSH Server 镜像（仅当前架构） +docker build --progress=plain -t nexent/nexent-ubuntu-terminal-conda -f deploy/images/dockerfiles/terminal/Dockerfile --build-arg TERMINAL_VARIANT=conda . ``` ## 🔧 镜像说明 ### 主应用镜像 (nexent/nexent) - 包含后端 API 服务 -- 基于 `make/main/Dockerfile` 构建 +- 基于 `deploy/images/dockerfiles/main/Dockerfile` 构建 - 提供核心的智能体服务 ### 数据处理镜像 (nexent/nexent-data-process) - 包含数据处理服务 -- 基于 `make/data_process/Dockerfile` 构建 +- 基于 `deploy/images/dockerfiles/data-process/Dockerfile` 构建 - 处理文档解析和向量化 ### 前端镜像 (nexent/nexent-web) - 包含 Next.js 前端应用 -- 基于 `make/web/Dockerfile` 构建 +- 基于 `deploy/images/dockerfiles/web/Dockerfile` 构建 - 提供用户界面 ### 文档镜像 (nexent/nexent-docs) - 包含 Vitepress 文档站点 -- 基于 `make/docs/Dockerfile` 构建 +- 基于 `deploy/images/dockerfiles/docs/Dockerfile` 构建 - 提供项目文档和 API 参考 ### MCP Server 镜像 (nexent/nexent-mcp) - 包含 MCP (Model Context Protocol) 代理服务 -- 基于 `make/mcp/Dockerfile` 构建 +- 基于 `deploy/images/dockerfiles/mcp/Dockerfile` 构建 - 为 AI 模型集成提供 MCP 服务器功能 #### 预装工具和特性 -- **Python 环境**: Python 3.10 + pip +- **Python 环境**: Python 3.11 + pip - **MCP Proxy**: mcp-proxy 包用于协议处理 - **Node.js**: Node.js 20.17.0 包含 npm - **架构支持**: linux/amd64, linux/arm64 -- **基础镜像**: python:3.10-slim +- **基础镜像**: python:3.11-slim ### OpenSSH Server 镜像 (nexent/nexent-ubuntu-terminal) - 基于 Ubuntu 24.04 的 SSH 服务器容器 -- 基于 `make/terminal/Dockerfile` 构建 -- 预装 Conda、Python、Git 等开发工具 -- 支持 SSH 密钥认证，用户名为 `linuxserver.io` -- 提供完整的开发环境 +- 基于 `deploy/images/dockerfiles/terminal/Dockerfile` 构建 +- 默认预装 OpenSSH、Python、pip、venv、Git、Curl、Wget +- `TERMINAL_VARIANT=conda` 额外预装 Miniconda、Vim 和编译工具链 +- 以 root 用户运行，支持 root 登录和密码认证 #### 预装工具和特性 -- **Python 环境**: Python 3 + pip + virtualenv -- **Conda 管理**: Miniconda3 环境管理 -- **开发工具**: Git、Vim、Nano、Curl、Wget -- **构建工具**: build-essential、Make -- **SSH 服务**: 端口 2222，禁用 root 登录和密码认证 -- **用户权限**: `linuxserver.io` 用户具有 sudo 权限（无需密码） -- **时区设置**: Asia/Shanghai -- **安全配置**: SSH 密钥认证，会话超时 60 分钟 +- **Python 环境**: Python 3 + pip + venv +- **Conda 管理**: 仅 `conda` 变体包含 Miniconda3 +- **开发工具**: Git、Curl、Wget；`conda` 变体额外包含 Vim 和 build-essential +- **SSH 服务**: 容器端口 22，允许 root 登录和密码认证 ## 🏷️ 标签策略 @@ -127,7 +173,7 @@ docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f make/terminal/ ### 构建文档镜像 ```bash -docker build -t nexent/nexent-docs -f make/docs/Dockerfile . +docker build -t nexent/nexent-docs -f deploy/images/dockerfiles/docs/Dockerfile . ``` ### 运行文档容器 @@ -163,8 +209,32 @@ docker rm nexent-docs 构建完成后，可以进入 `docker` 目录使用部署脚本启动本地镜像： ```bash -cd docker -bash deploy.sh --image-source local-latest +bash deploy.sh docker --image-source local-latest +``` + +> `local-latest` 会使用本地 `latest` Nexent 应用镜像并避免重新拉取这些镜像，无需修改 `deploy/docker/deploy.sh`。 + +### 将本地镜像打包为离线部署包 + +构建本地 `latest` 镜像后，可以使用离线打包脚本把镜像和部署资源打包： + +```bash +bash deploy/offline/build_offline_package.sh \ + --target docker \ + --version latest \ + --platform amd64 \ + --components infrastructure,application,data-process,supabase \ + --image-source local-latest \ + --compress true \ + --output-dir offline-package/docker-local ``` -> `local-latest` 会使用本地 `latest` Nexent 应用镜像并避免重新拉取这些镜像，无需修改 `docker/deploy.sh`。 +使用 `--version latest` 或 `--image-source local-latest` 时，脚本会使用本地 Nexent 应用镜像，并跳过这些 `latest` 标签的拉取。将包复制到目标机器后，可加载镜像并部署： + +```bash +cd offline-package/docker-local +bash deploy.sh --load-images docker \ + --version latest \ + --components infrastructure,application,data-process,supabase \ + --image-source local-latest +``` diff --git a/doc/docs/zh/developer-guide/environment-setup.md b/doc/docs/zh/developer-guide/environment-setup.md index cc98ff58a..aeca848b6 100644 --- a/doc/docs/zh/developer-guide/environment-setup.md +++ b/doc/docs/zh/developer-guide/environment-setup.md @@ -22,8 +22,7 @@ title: 环境准备 ```bash # 在项目根目录的 docker 目录执行 -cd docker -./deploy.sh --components infrastructure --port-policy development +bash deploy.sh docker --components infrastructure --port-policy development ``` :::: info 重要提示 diff --git a/doc/docs/zh/quick-start/installation.md b/doc/docs/zh/quick-start/installation.md index 6d3538b90..e2991c71b 100644 --- a/doc/docs/zh/quick-start/installation.md +++ b/doc/docs/zh/quick-start/installation.md @@ -18,17 +18,17 @@ ```bash git clone https://github.com/ModelEngine-Group/nexent.git -cd nexent/docker +cd nexent ``` -> **💡 提示**: `deploy.sh` 会在 `docker/.env` 不存在时自动从 `.env.example` 复制一份。若无特殊需求，可直接部署；若需要配置语音模型（STT/TTS），请部署前或部署后修改 `docker/.env` 中的相关参数。 +> **💡 提示**: `deploy.sh` 使用项目根目录 `.env` 作为运行配置。已有 `.env` 会原样保留；如果不存在，会优先复用已有 `docker/.env`，再回退到 `.env.example` 或 `docker/.env.example`。若需要配置语音模型（STT/TTS），请部署前或部署后修改 `.env` 中的相关参数。 ### 2. 部署选项 运行以下命令开始部署： ```bash -bash deploy.sh +bash deploy.sh docker ``` 执行此命令后，系统会通过 Bash TUI 选择部署参数。可使用方向键或 `j/k` 移动，空格切换多选项，回车确认，`b`/Backspace 返回上一步，`q` 退出。 @@ -36,8 +36,8 @@ bash deploy.sh **组件组合:** - **infrastructure（必选）**: Elasticsearch、PostgreSQL、Redis、MinIO - **application（默认选中，可取消）**: config、runtime、mcp、northbound、web -- **data-process（可选）**: 数据处理服务 -- **supabase（可选）**: 启用用户、租户和认证能力 +- **data-process（默认选中，可选）**: 数据处理服务 +- **supabase（默认选中，可选）**: 启用用户、租户和认证能力 - **terminal（可选）**: 启用 OpenSSH 终端工具 - **monitoring（可选）**: 启用观测组件，选择后会继续选择 provider @@ -54,19 +54,19 @@ bash deploy.sh ```bash # 默认组件组合，development 端口策略，标准镜像源 -bash deploy.sh --components infrastructure,application --port-policy development --image-source general +bash deploy.sh docker --components infrastructure,application,data-process,supabase --port-policy development --image-source general # 启用用户/租户能力、数据处理和终端工具 -bash deploy.sh --components infrastructure,application,supabase,data-process,terminal +bash deploy.sh docker --components infrastructure,application,data-process,supabase,terminal # 使用中国大陆镜像源 -bash deploy.sh --image-source mainland +bash deploy.sh docker --image-source mainland # 使用本地 latest 镜像 -bash deploy.sh --image-source local-latest +bash deploy.sh docker --image-source local-latest ``` -部署成功后，非敏感部署选项会保存到 `docker/deploy.options`。下次交互部署时可选择复用本地配置或重新全量配置。 +部署成功后，非敏感部署选项会保存到 `deploy/docker/deploy.options`。下次交互部署时可选择复用本地配置或重新全量配置。 #### ⚠️ 重要提示 @@ -148,7 +148,52 @@ Nexent 使用 Docker volumes 进行数据持久化： 默认 `dataDir` 为 `./volumes`（可在 `.env` 中配置 `ROOT_DIR`）。 -卸载由 `docker/uninstall.sh` 负责。默认交互询问是否删除持久化数据；也可使用 `--delete-volumes true|false`、`--remove-volumes`、`--keep-volumes`，或使用 `bash uninstall.sh delete-all` 删除容器和持久化数据。 +### 卸载 Docker 部署 + +请在仓库根目录使用统一卸载入口： + +```bash +# 停止并删除容器；是否删除持久化数据由交互确认 +bash uninstall.sh docker + +# 非交互卸载并保留数据 +bash uninstall.sh docker --keep-volumes + +# 删除 Docker volumes 和 ROOT_DIR 下的 Nexent 数据 +bash uninstall.sh docker --delete-volumes true + +# 完整清理：容器和持久化数据都会删除 +bash uninstall.sh docker delete-all +``` + +Docker 卸载脚本会读取 `.env` 中的 `ROOT_DIR` 并清理 Compose 资源。删除数据时会移除 `postgresql`、`elasticsearch`、`redis`、`minio`、`volumes`、`openssh-server`、`scripts`、`skills` 等服务目录；如果后续要复用已有数据，请选择保留 volumes。 + +### 离线镜像包 + +需要把镜像和部署脚本搬到离线机器时，可使用 `deploy/offline/build_offline_package.sh`： + +```bash +bash deploy/offline/build_offline_package.sh \ + --target docker \ + --version v2.2.1 \ + --platform amd64 \ + --components infrastructure,application,data-process,supabase \ + --image-source general \ + --compress true \ + --output-dir offline-package/docker +``` + +包目录会包含 `images/*.tar`、`load-images.sh`、`deploy.sh`、`uninstall.sh`、`manifest.yaml`、`checksums.txt`、`.env.example` 和 `deploy/sql`，不会包含本地 `.env` 或 `deploy.options`。使用 `--compress true` 时，会在输出目录的父目录生成 `nexent-offline---.zip`。 + +在目标机器上部署时，请保持部署参数与 `manifest.yaml` 中的版本、组件和镜像源一致： + +```bash +cd offline-package/docker +bash deploy.sh --load-images docker \ + --version v2.2.1 \ + --components infrastructure,application,data-process,supabase \ + --image-source general +``` ## 🔌 端口映射 @@ -171,14 +216,14 @@ Nexent 使用 Docker volumes 进行数据持久化： ### 监控配置 -部署时在脚本交互界面中选择 `monitoring` 组件即可启用 OpenTelemetry 监控。脚本会同步更新 `docker/.env` 中的 `ENABLE_TELEMETRY`、`MONITORING_PROVIDER` 和 `MONITORING_DASHBOARD_URL`，并启动 `docker/docker-compose-monitoring.yml` 中对应的观测组件。 +部署时在脚本交互界面中选择 `monitoring` 组件即可启用 OpenTelemetry 监控。脚本会同步更新 `.env` 中的 `ENABLE_TELEMETRY`、`MONITORING_PROVIDER` 和 `MONITORING_DASHBOARD_URL`，并启动 `deploy/docker/compose/docker-compose-monitoring.yml` 中对应的观测组件。 ```bash -cd nexent/docker -bash deploy.sh +cd nexent +bash deploy.sh docker ``` -如果本地已有 `docker/deploy.options`，脚本会询问是否复用本地配置。请选择重新配置/覆盖本地配置，然后在组件选择界面勾选 `monitoring`，再在 provider 选择界面手动选择 `grafana`、`phoenix`、`langfuse`、`langsmith`、`zipkin` 或 `otlp`。 +如果本地已有 `deploy/docker/deploy.options`，脚本会询问是否复用本地配置。请选择重新配置/覆盖本地配置，然后在组件选择界面勾选 `monitoring`，再在 provider 选择界面手动选择 `grafana`、`phoenix`、`langfuse`、`langsmith`、`zipkin` 或 `otlp`。 支持的 provider： @@ -194,7 +239,7 @@ bash deploy.sh 如需调整端口、镜像版本或 Langfuse 初始账号，请先复制并编辑监控环境变量： ```bash -cp docker/monitoring/monitoring.env.example docker/monitoring/monitoring.env +cp deploy/docker/assets/monitoring/monitoring.env.example deploy/docker/assets/monitoring/monitoring.env ``` 常用变量： @@ -207,7 +252,7 @@ cp docker/monitoring/monitoring.env.example docker/monitoring/monitoring.env | `LANGFUSE_INIT_USER_EMAIL` / `LANGFUSE_INIT_USER_PASSWORD` | 本地 Langfuse 初始管理员账号 | | `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD` | 本地 Grafana 管理员账号 | -选择 `langsmith` provider 前，请先在 `docker/monitoring/monitoring.env` 中配置 `LANGSMITH_API_KEY`。如果只需要连接已有外部 Collector，也可以在 `docker/.env` 中调整 OTLP 目标地址： +选择 `langsmith` provider 前，请先在 `deploy/docker/assets/monitoring/monitoring.env` 中配置 `LANGSMITH_API_KEY`。如果只需要连接已有外部 Collector，也可以在 `.env` 中调整 OTLP 目标地址： ```bash ENABLE_TELEMETRY=true @@ -224,10 +269,10 @@ MONITORING_DASHBOARD_URL= OAuth 登录依赖 `supabase` 组件。启用第三方登录时，请同时部署 `supabase`，并将 `OAUTH_CALLBACK_BASE_URL` 设置为浏览器可访问的 Nexent Web 地址。 ```bash -bash deploy.sh --components infrastructure,application,supabase +bash deploy.sh docker --components infrastructure,application,supabase ``` -Docker 部署在 `docker/.env` 中配置 OAuth： +Docker 部署在 `.env` 中配置 OAuth： ```bash # Web 入口地址。回调完整路径会自动拼接为： @@ -273,7 +318,7 @@ Provider 启用规则： CAS SSO 不依赖 `supabase`。启用 CAS 时，请将 `CAS_CALLBACK_BASE_URL` 设置为浏览器可访问的 Nexent Web 地址，且不要带结尾 `/`。`CAS_SERVER_URL` 是 CAS Server 根地址，也不要带结尾 `/`。 -Docker 部署在 `docker/.env` 中配置 CAS： +Docker 部署在 `.env` 中配置 CAS： ```bash CAS_ENABLED=true diff --git a/doc/docs/zh/quick-start/kubernetes-installation.md b/doc/docs/zh/quick-start/kubernetes-installation.md index 7229f1ea8..dbe44938d 100644 --- a/doc/docs/zh/quick-start/kubernetes-installation.md +++ b/doc/docs/zh/quick-start/kubernetes-installation.md @@ -27,7 +27,7 @@ kubectl get nodes ```bash git clone https://github.com/ModelEngine-Group/nexent.git -cd nexent/k8s/helm +cd nexent ``` ### 3. 部署 @@ -35,7 +35,7 @@ cd nexent/k8s/helm 运行部署脚本： ```bash -./deploy.sh +bash deploy.sh k8s ``` 执行此命令后，系统会通过 Bash TUI 选择配置选项。可使用方向键或 `j/k` 移动，空格切换多选项，回车确认，`b`/Backspace 返回上一步，`q` 退出。 @@ -43,8 +43,8 @@ cd nexent/k8s/helm **组件组合:** - **infrastructure（必选）**: Elasticsearch、PostgreSQL、Redis、MinIO - **application（默认选中，可取消）**: config、runtime、mcp、northbound、web -- **data-process（可选）**: 数据处理服务 -- **supabase（可选）**: 启用用户、租户和认证能力 +- **data-process（默认选中，可选）**: 数据处理服务 +- **supabase（默认选中，可选）**: 启用用户、租户和认证能力 - **terminal（可选）**: 启用 OpenSSH 终端工具 - **monitoring（可选）**: 启用观测组件，选择后会继续选择 provider @@ -57,7 +57,9 @@ cd nexent/k8s/helm - **mainland**: 使用中国大陆镜像源 - **local-latest**: 使用本地 `latest` 镜像，并将 Nexent 应用镜像的拉取策略设为本地优先 -部署成功后，非敏感部署选项会保存到 `k8s/helm/deploy.options`。下次交互部署时可选择复用本地配置或重新全量配置。 +Kubernetes 使用与 Docker 相同的项目根目录 `.env`。已有 `.env` 会原样保留；如果不存在，部署脚本会优先复用已有 `docker/.env`，再回退到 `.env.example` 或 `docker/.env.example`。 + +部署成功后，非敏感部署选项会保存到 `deploy/k8s/deploy.options`。下次交互部署时可选择复用本地配置或重新全量配置。 ### ⚠️ 重要提示 @@ -80,7 +82,7 @@ kubectl exec -it -n nexent deploy/nexent-postgresql -- psql -U root -d nexent -c "DELETE FROM nexent.user_tenant_t WHERE user_id='your_user_id';" # Step 3: 重新部署并记录 su 账号密码 -./deploy.sh +bash deploy.sh k8s ``` ### 4. 访问您的安装 @@ -155,44 +157,99 @@ Nexent 使用 PersistentVolume 进行数据持久化： | Redis | nexent-redis-pv | `/var/lib/nexent-data/nexent-redis` | | MinIO | nexent-minio-pv | `/var/lib/nexent-data/nexent-minio` | | Supabase DB（选择 supabase 时）| nexent-supabase-db-pv | `/var/lib/nexent-data/nexent-supabase-db` | +| 共享工作区 | nexent-workspace-pv | `/var/lib/nexent` | +| 共享技能目录 | nexent-skills-pv | `/var/lib/nexent-data/skills` | + +卸载 Helm release 默认不会删除本地 hostPath 数据。可使用 `bash uninstall.sh k8s --delete-local-data true` 删除 `/var/lib/nexent`、`/var/lib/nexent-data/skills` 和 `/var/lib/nexent-data/nexent-*` 下的 Nexent 本地卷内容，使用 `--keep-local-data` 显式保留。 + +### 卸载 Kubernetes 部署 + +请在仓库根目录使用统一卸载入口： + +```bash +# 删除 Helm release；交互模式会询问是否删除 namespace 和本地数据 +bash uninstall.sh k8s + +# 仅清理 Helm release 状态，适合修复卡住的发布 +bash uninstall.sh k8s clean + +# 删除 Helm release 和 namespace，但保留本地 hostPath 数据 +bash uninstall.sh k8s delete --keep-local-data + +# 卸载后删除已知本地 hostPath 数据 +bash uninstall.sh k8s --delete-local-data true + +# 完整清理：Helm release、namespace 和本地 hostPath 数据都会删除 +bash uninstall.sh k8s delete-all +``` -卸载 Helm release 默认不会删除本地 hostPath 数据。可使用 `./uninstall.sh --delete-local-data true` 删除 `/var/lib/nexent-data/nexent-*` 下的 Nexent 本地卷内容，使用 `--keep-local-data` 显式保留。 +`--delete-data` 和 `--delete-volumes` 是兼容 Helm 管理资源的参数；本地盘数据请使用 `--delete-local-data` 或 `--keep-local-data` 控制。`delete-all --keep-local-data` 会删除 namespace，但保留本地卷内容。 + +### 离线镜像包 + +可在仓库根目录构建 Kubernetes 离线包： + +```bash +bash deploy/offline/build_offline_package.sh \ + --target k8s \ + --version v2.2.1 \ + --platform amd64 \ + --components infrastructure,application,data-process,supabase \ + --image-source general \ + --compress true \ + --output-dir offline-package/k8s +``` + +包内包含镜像 tar、`load-images.sh`、根目录部署/卸载入口、Kubernetes Helm 资源、SQL 文件、`manifest.yaml` 和 `checksums.txt`。使用 `--compress true` 时，会在输出目录的父目录生成 `nexent-offline---.zip`。如果是单节点、Docker 作为容器运行时的集群，可以直接加载并部署： + +```bash +cd offline-package/k8s +bash deploy.sh --load-images k8s \ + --version v2.2.1 \ + --components infrastructure,application,data-process,supabase \ + --image-source general +``` + +多节点集群需要在每个可能运行 Nexent Pod 的节点上加载镜像，或将镜像推送到集群可访问的内部镜像仓库，再使用匹配的镜像参数部署。 ## 🔧 部署命令 ```bash # 交互式部署 -./deploy.sh +bash deploy.sh k8s # 非交互式部署默认组件 -./deploy.sh --components infrastructure,application --port-policy development --image-source general +bash deploy.sh k8s --components infrastructure,application,data-process,supabase --port-policy development --image-source general # 启用用户/租户能力、数据处理和终端工具 -./deploy.sh --components infrastructure,application,supabase,data-process,terminal +bash deploy.sh k8s --components infrastructure,application,data-process,supabase,terminal # 使用中国大陆镜像源部署 -./deploy.sh --image-source mainland +bash deploy.sh k8s --image-source mainland # 使用本地 latest 镜像 -./deploy.sh --image-source local-latest +bash deploy.sh k8s --image-source local-latest + +# 使用 --sc 简写指定 StorageClass +bash deploy.sh k8s --sc fast-storage # 仅清理 Helm 状态（修复卡住的发布） -./uninstall.sh clean +bash uninstall.sh k8s clean # 卸载，默认保留本地数据；交互确认是否删除 namespace 和本地数据 -./uninstall.sh +bash uninstall.sh k8s # 卸载并删除 namespace -./uninstall.sh --delete-namespace true +bash uninstall.sh k8s --delete-namespace true # 卸载并删除本地 hostPath 数据 -./uninstall.sh --delete-local-data true +bash uninstall.sh k8s --delete-local-data true # 完全卸载，包括 namespace 和本地 hostPath 数据 -./uninstall.sh delete-all +bash uninstall.sh k8s delete-all # 完全卸载但保留本地 hostPath 数据 -./uninstall.sh delete-all --keep-local-data +bash uninstall.sh k8s delete-all --keep-local-data ``` ## 🔧 高级配置 @@ -202,11 +259,11 @@ Nexent 使用 PersistentVolume 进行数据持久化： Kubernetes 部署通过脚本交互界面中的 `monitoring` 组件启用监控。部署脚本会生成运行时 Helm values，设置 `global.monitoring.enabled`、`global.monitoring.provider`、`global.monitoring.dashboardUrl`，并启用 `nexent-monitoring` 子 Chart。 ```bash -cd nexent/k8s/helm -./deploy.sh +cd nexent +bash deploy.sh k8s ``` -如果本地已有 `k8s/helm/deploy.options`，脚本会询问是否复用本地配置。请选择重新配置/覆盖本地配置，然后在组件选择界面勾选 `monitoring`，再在 provider 选择界面手动选择 `grafana`、`phoenix`、`langfuse`、`langsmith`、`zipkin` 或 `otlp`。 +如果本地已有 `deploy/k8s/deploy.options`，脚本会询问是否复用本地配置。请选择重新配置/覆盖本地配置，然后在组件选择界面勾选 `monitoring`，再在 provider 选择界面手动选择 `grafana`、`phoenix`、`langfuse`、`langsmith`、`zipkin` 或 `otlp`。 支持的 provider： @@ -219,7 +276,7 @@ cd nexent/k8s/helm | `grafana` | 本地 Grafana + Tempo | `http://localhost:30002/d/nexent-llm-agent/nexent-agent-trace-monitoring?orgId=1` | | `zipkin` | 本地 Zipkin | `http://localhost:30011` | -选择 `langsmith` provider 前，请先在 `k8s/helm/nexent/values.yaml` 中配置 `global.monitoring.langsmithApiKey` 和 `global.monitoring.langsmithProject`。如需修改本地 Grafana、Langfuse 或各 Dashboard 的端口，也建议先在 values 文件中调整，再通过部署脚本重新配置并手动选择 `monitoring`。 +选择 `langsmith` provider 前，请先在 `deploy/deploy/k8s/helm/nexent/values.yaml` 中配置 `global.monitoring.langsmithApiKey` 和 `global.monitoring.langsmithProject`。如需修改本地 Grafana、Langfuse 或各 Dashboard 的端口，也建议先在 values 文件中调整，再通过部署脚本重新配置并手动选择 `monitoring`。 常用 Helm values： @@ -248,7 +305,7 @@ kubectl get svc -n nexent | grep -E 'otel|phoenix|grafana|zipkin|langfuse' OAuth 登录依赖 `supabase` 组件。启用第三方登录时，请同时部署 `supabase`，并将 `config.oauth.callbackBaseUrl` 设置为浏览器可访问的 Nexent Web 地址。 ```bash -./deploy.sh --components infrastructure,application,supabase +bash deploy.sh k8s --components infrastructure,application,supabase ``` Kubernetes 部署通过 `nexent-common` 的 `config.oauth.*` values 写入后端环境变量： diff --git a/doc/docs/zh/quick-start/kubernetes-upgrade-guide.md b/doc/docs/zh/quick-start/kubernetes-upgrade-guide.md index f2ec9226a..10d5d9f05 100644 --- a/doc/docs/zh/quick-start/kubernetes-upgrade-guide.md +++ b/doc/docs/zh/quick-start/kubernetes-upgrade-guide.md @@ -14,7 +14,7 @@ 更新之前，先记录下当前部署的版本和数据目录信息。 -- 当前部署版本信息的位置：`backend/consts/const.py` 中的 `APP_VERSION` +- 当前部署版本信息的位置：根目录 `VERSION` - 本地卷目录信息的位置：各 Helm 子 chart 的 `storage.hostPath`，默认位于 `/var/lib/nexent-data/nexent-*` **git 方式下载的代码** @@ -28,15 +28,14 @@ git pull **zip 包等方式下载的代码** 1. 需要去 GitHub 上重新下载一份最新代码，并解压缩。 -2. 将之前执行部署脚本目录下 `k8s/helm` 目录中的 `deploy.options` 文件拷贝到新代码目录的 `k8s/helm` 目录中。（如果不存在该文件则忽略此步骤）。 +2. 将之前部署目录 `deploy/k8s` 下的 `deploy.options` 文件拷贝到新代码目录的 `deploy/k8s` 目录中。（如果不存在该文件则忽略此步骤）。 ## 🔄 步骤二：执行升级 -进入更新后代码目录的 `k8s/helm` 目录，执行部署脚本： +在更新后的代码仓库根目录执行 Kubernetes 部署入口： ```bash -cd k8s/helm -./deploy.sh +bash deploy.sh k8s ``` 脚本会自动检测您之前保存的部署设置（组件组合、端口策略、镜像来源等）。如果 `deploy.options` 文件不存在，系统会提示您输入配置信息。 @@ -55,79 +54,11 @@ cd k8s/helm --- -## 🗄️ 手动更新数据库 +## 🗄️ 数据库迁移 -升级时如果存在部分 SQL 文件执行失败，或需要手动执行增量 SQL 脚本时，可以通过以下方法进行更新。 +SQL 增量不再手动执行。Kubernetes 中只有 `nexent-config` 启动时会通过 `deploy/common/run-sql-migrations.sh` 自动按文件名顺序检查并执行 `deploy/sql/migrations/` 下的 `*.sql` 文件；其他后端服务只等待迁移记录达到目标状态。部署脚本会将 `deploy/sql` 渲染到共享 SQL ConfigMap，并挂载到 `/opt/nexent/sql`，因此只修改 SQL 时重新执行部署即可，不需要重新构建镜像。 -### 📋 查找 SQL 脚本 - -SQL 迁移脚本位于仓库的： - -``` -docker/sql/ -``` - -请查看 [升级指南](./upgrade-guide.md) 或版本发布说明，确认需要执行哪些 SQL 脚本。 - -### ✅ 方法一：使用 SQL 编辑器（推荐） - -1. 打开 SQL 编辑器，新建 PostgreSQL 连接。 -2. 从正在运行的 PostgreSQL Pod 中获取连接信息： - - ```bash - # 获取 PostgreSQL Pod 名称 - kubectl get pods -n nexent -l app=nexent-postgresql - - # 端口转发以便本地访问 PostgreSQL - kubectl port-forward svc/nexent-postgresql 5433:5432 -n nexent & - ``` - -3. 连接信息： - - Host: `localhost` - - Port: `5433`（转发的端口） - - Database: `nexent` - - User: `root` - - Password: 可在 `k8s/helm/nexent/charts/nexent-common/values.yaml` 中查看 - -4. 填写连接信息后测试连接，确认成功后可在 `nexent` schema 中查看所有表。 -5. 按版本顺序执行所需的 SQL 文件。 - -> ⚠️ 注意事项 -> - 升级前请备份数据库，生产环境尤为重要。 -> - SQL 脚本需按时间顺序执行，避免依赖冲突。 - -### 🧰 方法二：使用 kubectl exec（无需客户端） - -通过 stdin 重定向直接在主机上执行 SQL 脚本： - -1. 获取 PostgreSQL Pod 名称： - - ```bash - kubectl get pods -n nexent -l app=nexent-postgresql -o jsonpath='{.items[0].metadata.name}' - ``` - -2. 直接从主机执行 SQL 文件： - - ```bash - kubectl exec -i -n nexent -- psql -U root -d nexent < ./sql/v1.1.1_1030-update.sql - ``` - - 或者如果想交互式查看输出： - - ```bash - cat ./sql/v1.1.1_1030-update.sql | kubectl exec -i -n nexent -- psql -U root -d nexent - ``` - -**示例 - 依次执行多个 SQL 文件：** - -```bash -# 获取 PostgreSQL Pod 名称 -POSTGRES_POD=$(kubectl get pods -n nexent -l app=nexent-postgresql -o jsonpath='{.items[0].metadata.name}') - -# 按顺序执行 SQL 文件 -kubectl exec -i $POSTGRES_POD -n nexent -- psql -U root -d nexent < ./sql/v1.8.0_xxxxx-update.sql -kubectl exec -i $POSTGRES_POD -n nexent -- psql -U root -d nexent < ./sql/v2.0.0_0314_add_context_skill_t.sql -``` +迁移脚本使用 SQL 文件名作为 `nexent.schema_migrations` 中的迁移 ID。已记录且 checksum 相同会跳过；已记录但 checksum 变化时会重新执行同名 SQL，并更新 checksum、执行时间、应用版本和源文件路径。 > 💡 提示 > - 执行前建议先备份数据库： @@ -137,13 +68,7 @@ kubectl exec -i $POSTGRES_POD -n nexent -- psql -U root -d nexent < ./sql/v2.0.0 kubectl exec nexent/$POSTGRES_POD -n nexent -- pg_dump -U root nexent > backup_$(date +%F).sql ``` -> - 对于 Supabase 数据库（选择 `supabase` 组件时），请使用 `nexent-supabase-db` Pod： - - ```bash - SUPABASE_POD=$(kubectl get pods -n nexent -l app=nexent-supabase-db -o jsonpath='{.items[0].metadata.name}') - kubectl cp docker/sql/xxx.sql nexent/$SUPABASE_POD:/tmp/update.sql - kubectl exec -it nexent/$SUPABASE_POD -n nexent -- psql -U postgres -f /tmp/update.sql - ``` +> - Supabase 初始化 SQL 由部署脚本从 `deploy/sql/supabase/` 渲染到 Helm values，不需要手动复制执行。 --- @@ -163,9 +88,7 @@ kubectl logs -n nexent -l app=nexent-config --tail=100 kubectl logs -n nexent -l app=nexent-web --tail=100 ``` -### 手动 SQL 更新后重启服务（如需要） - -如果您手动执行了 SQL 脚本，需要重启受影响的服务： +### 迁移重试后重启服务 ```bash kubectl rollout restart deployment/nexent-config -n nexent @@ -175,6 +98,5 @@ kubectl rollout restart deployment/nexent-runtime -n nexent ### 重新初始化 Elasticsearch（如需要） ```bash -cd k8s/helm -bash init-elasticsearch.sh +bash deploy/k8s/init-elasticsearch.sh ``` diff --git a/doc/docs/zh/quick-start/upgrade-guide.md b/doc/docs/zh/quick-start/upgrade-guide.md index 4f8b429e0..1a6716e3d 100644 --- a/doc/docs/zh/quick-start/upgrade-guide.md +++ b/doc/docs/zh/quick-start/upgrade-guide.md @@ -14,8 +14,8 @@ 更新之前，先记录下当前部署的版本和数据目录 -- 当前部署版本信息的位置：`backend/consts/const.py`中的 APP_VERSION -- 数据目录信息的位置：`docker/.env`中的 ROOT_DIR +- 当前部署版本信息的位置：根目录 `VERSION` +- 数据目录信息的位置：`.env`中的 ROOT_DIR **git 方式下载的代码** @@ -31,17 +31,17 @@ git pull ## 🔄 步骤二：执行升级 -进入更新后代码目录的docker目录，执行升级脚本： +在更新后的代码仓库根目录执行 Docker 部署入口： ```bash -bash upgrade.sh +bash deploy.sh docker ``` 缺少 deploy.options 的情况下，会提示需要重新选择部署配置，例如组件组合、端口策略、镜像来源等。按照您之前的部署方式重新选择即可。 > 💡 提示 -> - 若 `docker/.env` 不存在，部署脚本会从 `.env.example` 自动复制一份。 -> - 若需配置语音模型（STT/TTS），请在 `docker/.env` 中补充相关变量，我们将尽快提供前端配置入口。 +> - 已有 `.env` 会原样保留；如果不存在，部署脚本会优先复用已有 `docker/.env`，再回退到 `.env.example` 或 `docker/.env.example`。 +> - 若需配置语音模型（STT/TTS），请在 `.env` 中补充相关变量，我们将尽快提供前端配置入口。 ## 🌐 步骤三：验证部署 @@ -80,74 +80,12 @@ docker system prune -af --- -### 🗄️ 手动更新数据库 +### 🗄️ 数据库迁移 -升级时如果存在部分 sql 文件执行失败，则可以手动执行更新。 +SQL 增量不再手动执行。Docker 中只有 `nexent-config` 启动时会通过 `deploy/common/run-sql-migrations.sh` 自动按文件名顺序检查并执行 `deploy/sql/migrations/` 下的 `*.sql` 文件；其他后端容器只等待迁移记录达到目标状态。SQL 会从 `deploy/sql` 挂载到 `/opt/nexent/sql`，因此只修改 SQL 时重新执行部署即可，不需要重新构建镜像。 -#### ✅ 方法一：使用 SQL 编辑器（推荐） - -1. 打开 SQL 编辑器，新建 PostgreSQL 连接。 -2. 在 `/nexent/docker/.env` 中找到以下信息： - - Host - - Port - - Database - - User - - Password -3. 填写连接信息后测试连接，确认成功后可在 `nexent` schema 中查看所有表。 -4. 新建查询窗口。 -5. 打开 `/nexent/docker/sql` 目录，通过失败的sql文件查看 SQL 脚本。 -6. 将失败的sql文件和后续版本的sql文件依次执行。 - -> ⚠️ 注意事项 -> - 升版本前请备份数据库，生产环境尤为重要。 -> - SQL 脚本需按时间顺序执行，避免依赖冲突。 -> - `.env` 变量可能命名为 `POSTGRES_HOST`、`POSTGRES_PORT` 等，请在客户端对应填写。 - -#### 🧰 方法二：命令行执行（无需客户端） - -1. 进入 Docker 目录： - - ```bash - cd nexent/docker - ``` - -2. 从 `.env` 中获取数据库连接信息，例如： - - ```bash - POSTGRES_HOST=localhost - POSTGRES_PORT=5432 - POSTGRES_DB=nexent - POSTGRES_USER=root - POSTGRES_PASSWORD=your_password - ``` - -3. 通过容器执行 SQL 脚本（示例）： - - ```bash - # 我们需要执行以下命令（请注意替换占位符中的变量） - docker exec -i nexent-postgresql psql -U [YOUR_POSTGRES_USER] -d [YOUR_POSTGRES_DB] < ./sql/v1.1.1_1030-update.sql - docker exec -i nexent-postgresql psql -U [YOUR_POSTGRES_USER] -d [YOUR_POSTGRES_DB] < ./sql/v1.1.2_1105-update.sql - ``` - - 请根据自己的部署版本，按版本顺序执行对应脚本。 +迁移脚本使用 SQL 文件名作为 `nexent.schema_migrations` 中的迁移 ID。已记录且 checksum 相同会跳过；已记录但 checksum 变化时会重新执行同名 SQL，并更新 checksum、执行时间、应用版本和源文件路径。 > 💡 提示 -> - 若 `.env` 中定义了数据库变量，可先导入： -> -> **Windows PowerShell:** -> ```powershell -> Get-Content .env | Where-Object { $_ -notmatch '^#' -and $_ -match '=' } | ForEach-Object { $key, $value = $_ -split '=', 2; [Environment]::SetEnvironmentVariable($key.Trim(), $value.Trim(), 'Process') } -> ``` -> -> **Linux/WSL:** -> ```bash -> export $(grep -v '^#' .env | xargs) -> # 或使用 set -a 自动导出所有变量 -> set -a; source .env; set +a -> ``` -> -> - 执行前建议先备份： -> -> ```bash -> docker exec -i nexent-postgres pg_dump -U [YOUR_POSTGRES_USER] [YOUR_POSTGRES_DB] > backup_$(date +%F).sql -> ``` +> - 升级前请备份数据库，生产环境尤为重要。 +> - 如果服务启动失败，请查看后端容器日志中的 `[sql-migrations]` 记录。 diff --git a/doc/docs/zh/sdk/monitoring.md b/doc/docs/zh/sdk/monitoring.md index 2483b505b..da2f9e365 100644 --- a/doc/docs/zh/sdk/monitoring.md +++ b/doc/docs/zh/sdk/monitoring.md @@ -15,7 +15,7 @@ NexentAgent ──► OpenTelemetry SDK ──► OTLP Collector ──► Arize ## 快速启动 ```bash -cd docker +cd deploy/docker [ -f .env ] || cp .env.example .env cp monitoring/monitoring.env.example monitoring/monitoring.env @@ -44,7 +44,7 @@ MONITORING_PROVIDER=otlp | `grafana` | `./start-monitoring.sh --stack grafana` | Collector + Grafana + Tempo | 本地 Tempo trace 查询 | | `zipkin` | `./start-monitoring.sh --stack zipkin` | Collector + Zipkin | 本地 trace 查询 | -也可以在 `docker/monitoring/monitoring.env` 中设置默认形态： +也可以在 `deploy/docker/assets/monitoring/monitoring.env` 中设置默认形态： ```bash MONITORING_PROVIDER=phoenix @@ -55,7 +55,7 @@ MONITORING_PROVIDER=phoenix Phoenix 本地部署使用 `arizephoenix/phoenix` 镜像，默认 UI 端口为 `6006`，gRPC OTLP 端口映射为 `4319`，数据持久化到 Docker volume `phoenix-data`。 ```bash -cd docker +cd deploy/docker ./start-monitoring.sh --stack phoenix ``` @@ -81,7 +81,7 @@ OTEL_EXPORTER_OTLP_METRICS_ENABLED=false Langfuse 本地部署使用 v3 架构：Web、Worker、Postgres、ClickHouse、MinIO、Redis。默认 UI 端口为 `3001`，初始化项目和 API Key 来自 `monitoring.env`。 ```bash -cd docker +cd deploy/docker ./start-monitoring.sh --stack langfuse ``` @@ -98,7 +98,7 @@ cd docker LangSmith 支持通过在线 OTLP endpoint 摄取 traces。Nexent 可以先把 OTLP 发到本地 Collector，再由 Collector 转发到 LangSmith，业务服务无需直接保存 LangSmith API Key。 ```bash -cd docker +cd deploy/docker vim monitoring/monitoring.env MONITORING_PROVIDER=langsmith @@ -126,7 +126,7 @@ LangSmith 当前配置只转发 traces，OTLP metrics 会留在 Collector debug Grafana 本地部署使用 Grafana Tempo 存储 traces，并启用 Tempo `metrics-generator` 的 `local-blocks` processor 支持 Grafana trace breakdown 中的 TraceQL metrics 查询。Collector 接收 Nexent 后端的 OTLP traces/metrics，其中 traces 通过 OTLP gRPC 转发到 Tempo；OTLP metrics 只进入 Collector debug pipeline，不提供独立指标存储或指标 dashboard。 ```bash -cd docker +cd deploy/docker ./start-monitoring.sh --stack grafana ``` @@ -152,7 +152,7 @@ Grafana 会自动预置 Tempo datasource，并加载 `Nexent Agent Trace Monitor Zipkin 本地部署使用 `openzipkin/zipkin` 镜像。Collector 接收 Nexent 后端的 OTLP traces/metrics，其中 traces 转发到 Zipkin v2 spans endpoint；OTLP metrics 当前只进入 Collector debug pipeline。 ```bash -cd docker +cd deploy/docker ./start-monitoring.sh --stack zipkin ``` @@ -435,11 +435,11 @@ service: 本地 Phoenix 和 Langfuse 分别使用独立 Collector 配置： -- `docker/monitoring/otel-collector-phoenix-config.yml` -- `docker/monitoring/otel-collector-langfuse-config.yml` -- `docker/monitoring/otel-collector-langsmith-config.yml` +- `deploy/docker/assets/monitoring/otel-collector-phoenix-config.yml` +- `deploy/docker/assets/monitoring/otel-collector-langfuse-config.yml` +- `deploy/docker/assets/monitoring/otel-collector-langsmith-config.yml` -基础 debug 配置见 `docker/monitoring/otel-collector-config.yml`。 +基础 debug 配置见 `deploy/docker/assets/monitoring/otel-collector-config.yml`。 ## 优雅降级 diff --git a/doc/docs/zh/sdk/opentelemetry-design.md b/doc/docs/zh/sdk/opentelemetry-design.md index 2f8f0a678..46093c633 100644 --- a/doc/docs/zh/sdk/opentelemetry-design.md +++ b/doc/docs/zh/sdk/opentelemetry-design.md @@ -376,7 +376,7 @@ Zipkin 当前本地形态只转发 traces；metrics 进入 Collector debug pipel 启动命令： ```bash -cd docker +cd deploy/docker ./start-monitoring.sh --stack otlp ./start-monitoring.sh --stack phoenix ./start-monitoring.sh --stack langfuse diff --git a/doc/docs/zh/user-guide/local-tools/terminal-tool.md b/doc/docs/zh/user-guide/local-tools/terminal-tool.md index b0e298319..eb624cbd1 100644 --- a/doc/docs/zh/user-guide/local-tools/terminal-tool.md +++ b/doc/docs/zh/user-guide/local-tools/terminal-tool.md @@ -33,7 +33,7 @@ SSH端口: 2222 ##### 方式B：本地构建镜像 ```bash # 本地构建Ubuntu Terminal镜像 -docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f make/terminal/Dockerfile . +docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f deploy/images/dockerfiles/terminal/Dockerfile . ``` > 📚 **详细构建说明**：参考 [Docker 构建指南](/zh/deployment/docker-build) 了解完整的镜像构建和推送流程。 @@ -44,8 +44,7 @@ docker build --progress=plain -t nexent/nexent-ubuntu-terminal -f make/terminal/ ```bash # 运行部署脚本 -cd docker -bash deploy.sh +bash deploy.sh docker --components infrastructure,application,data-process,supabase,terminal # 在脚本执行过程中选择： # 1. 部署模式：选择开发/生产/基础设施模式 diff --git a/docker/.env.beta b/docker/.env.beta deleted file mode 100644 index 2ce33754e..000000000 --- a/docker/.env.beta +++ /dev/null @@ -1,9 +0,0 @@ -NEXENT_IMAGE=nexent/nexent:beta -NEXENT_WEB_IMAGE=nexent/nexent-web:beta -NEXENT_DATA_PROCESS_IMAGE=nexent/nexent-data-process:beta - -ELASTICSEARCH_IMAGE=docker.elastic.co/elasticsearch/elasticsearch:8.17.4 -POSTGRESQL_IMAGE=postgres:15-alpine -REDIS_IMAGE=redis:alpine -MINIO_IMAGE=quay.io/minio/minio:RELEASE.2023-12-20T01-00-02Z -OPENSSH_SERVER_IMAGE=nexent/nexent-ubuntu-terminal:latest \ No newline at end of file diff --git a/docker/generate_env.sh b/docker/generate_env.sh deleted file mode 100755 index c6b20f0b1..000000000 --- a/docker/generate_env.sh +++ /dev/null @@ -1,276 +0,0 @@ -#!/bin/bash - -# Exit immediately if a command exits with a non-zero status -set -e -echo " 📁 Target .env location: docker/.env" - -# Function to copy and prepare .env file -prepare_env_file() { - echo " 📝 Preparing docker/.env file..." - - if [ -f ".env" ]; then - echo " ✅ Using existing docker/.env" - elif [ -f ".env.example" ]; then - echo " 📋 docker/.env not found, copying docker/.env.example..." - cp ".env.example" ".env" - echo " ✅ Created docker/.env from docker/.env.example" - else - echo " ❌ ERROR Neither docker/.env nor docker/.env.example exists in docker directory" - ERROR_OCCURRED=1 - return 1 - fi -} - -# Function to update .env file with generated keys -update_env_file() { - echo " 📝 Updating docker/.env file with generated keys..." - - if [ ! -f ".env" ]; then - echo " ❌ ERROR docker/.env file does not exist" - ERROR_OCCURRED=1 - return 1 - fi - - # Update or add MINIO_ACCESS_KEY - if grep -q "^MINIO_ACCESS_KEY=" .env; then - sed -i.bak "s~^MINIO_ACCESS_KEY=.*~MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY~" .env - else - echo "" >> .env - echo "# Generated MinIO Keys" >> .env - echo "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" >> .env - fi - - # Update or add MINIO_SECRET_KEY - if grep -q "^MINIO_SECRET_KEY=" .env; then - sed -i.bak "s~^MINIO_SECRET_KEY=.*~MINIO_SECRET_KEY=$MINIO_SECRET_KEY~" .env - else - echo "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" >> .env - fi - - # Update or add ELASTICSEARCH_API_KEY (only if it was generated successfully) - if [ -n "$ELASTICSEARCH_API_KEY" ]; then - if grep -q "^ELASTICSEARCH_API_KEY=" .env; then - sed -i.bak "s~^ELASTICSEARCH_API_KEY=.*~ELASTICSEARCH_API_KEY=$ELASTICSEARCH_API_KEY~" .env - else - echo "" >> .env - echo "# Generated Elasticsearch API Key" >> .env - echo "ELASTICSEARCH_API_KEY=$ELASTICSEARCH_API_KEY" >> .env - fi - fi - - # Update or add SSH credentials (only if they were set) - if [ -n "$SSH_USERNAME" ]; then - if grep -q "^SSH_USERNAME=" .env; then - sed -i.bak "s~^SSH_USERNAME=.*~SSH_USERNAME=$SSH_USERNAME~" .env - else - echo "" >> .env - echo "# SSH Terminal Tool Credentials" >> .env - echo "SSH_USERNAME=$SSH_USERNAME" >> .env - fi - fi - - if [ -n "$SSH_PASSWORD" ]; then - if grep -q "^SSH_PASSWORD=" .env; then - sed -i.bak "s~^SSH_PASSWORD=.*~SSH_PASSWORD=$SSH_PASSWORD~" .env - else - echo "SSH_PASSWORD=$SSH_PASSWORD" >> .env - fi - fi - echo " ✅ Generated keys updated successfully" - - # Force update development environment service URLs for localhost access - echo " 🔧 Updating service URLs for localhost development environment..." - - # ELASTICSEARCH_HOST - if grep -q "^ELASTICSEARCH_HOST=" .env; then - sed -i.bak "s~^ELASTICSEARCH_HOST=.*~ELASTICSEARCH_HOST=http://localhost:9210~" .env - else - echo "" >> .env - echo "# Development Environment URLs" >> .env - echo "ELASTICSEARCH_HOST=http://localhost:9210" >> .env - fi - - # Main Services - # CONFIG_SERVICE_URL - if grep -q "^CONFIG_SERVICE_URL=" .env; then - sed -i.bak "s~^CONFIG_SERVICE_URL=.*~CONFIG_SERVICE_URL=http://localhost:5010~" .env - else - echo "" >> .env - echo "# Main Services" >> .env - echo "CONFIG_SERVICE_URL=http://localhost:5010" >> .env - fi - - # RUNTIME_SERVICE_URL - if grep -q "^RUNTIME_SERVICE_URL=" .env; then - sed -i.bak "s~^RUNTIME_SERVICE_URL=.*~RUNTIME_SERVICE_URL=http://localhost:5014~" .env - else - echo "RUNTIME_SERVICE_URL=http://localhost:5014" >> .env - fi - - # ELASTICSEARCH_SERVICE - if grep -q "^ELASTICSEARCH_SERVICE=" .env; then - sed -i.bak "s~^ELASTICSEARCH_SERVICE=.*~ELASTICSEARCH_SERVICE=http://localhost:5010/api~" .env - else - echo "ELASTICSEARCH_SERVICE=http://localhost:5010/api" >> .env - fi - - # NEXENT_MCP_SERVER - if grep -q "^NEXENT_MCP_SERVER=" .env; then - sed -i.bak "s~^NEXENT_MCP_SERVER=.*~NEXENT_MCP_SERVER=http://localhost:5011~" .env - else - echo "NEXENT_MCP_SERVER=http://localhost:5011" >> .env - fi - - # DATA_PROCESS_SERVICE - if grep -q "^DATA_PROCESS_SERVICE=" .env; then - sed -i.bak "s~^DATA_PROCESS_SERVICE=.*~DATA_PROCESS_SERVICE=http://localhost:5012/api~" .env - else - echo "DATA_PROCESS_SERVICE=http://localhost:5012/api" >> .env - fi - - # NORTHBOUND_API_SERVER - if grep -q "^NORTHBOUND_API_SERVER=" .env; then - sed -i.bak "s~^NORTHBOUND_API_SERVER=.*~NORTHBOUND_API_SERVER=http://localhost:5013/api~" .env - else - echo "NORTHBOUND_API_SERVER=http://localhost:5013/api" >> .env - fi - - # MCP_MANAGEMENT_API - if grep -q "^MCP_MANAGEMENT_API=" .env; then - sed -i.bak "s~^MCP_MANAGEMENT_API=.*~MCP_MANAGEMENT_API=http://localhost:5015~" .env - else - echo "MCP_MANAGEMENT_API=http://localhost:5015" >> .env - fi - - # MINIO_ENDPOINT - if grep -q "^MINIO_ENDPOINT=" .env; then - sed -i.bak "s~^MINIO_ENDPOINT=.*~MINIO_ENDPOINT=http://localhost:9010~" .env - else - echo "MINIO_ENDPOINT=http://localhost:9010" >> .env - fi - - # REDIS_URL - if grep -q "^REDIS_URL=" .env; then - sed -i.bak "s~^REDIS_URL=.*~REDIS_URL=redis://localhost:6379/0~" .env - else - echo "REDIS_URL=redis://localhost:6379/0" >> .env - fi - - # REDIS_BACKEND_URL - if grep -q "^REDIS_BACKEND_URL=" .env; then - sed -i.bak "s~^REDIS_BACKEND_URL=.*~REDIS_BACKEND_URL=redis://localhost:6379/1~" .env - else - echo "REDIS_BACKEND_URL=redis://localhost:6379/1" >> .env - fi - - # POSTGRES_HOST - if grep -q "^POSTGRES_HOST=" .env; then - sed -i.bak "s~^POSTGRES_HOST=.*~POSTGRES_HOST=localhost~" .env - else - echo "POSTGRES_HOST=localhost" >> .env - fi - - # POSTGRES_PORT - if grep -q "^POSTGRES_PORT=" .env; then - sed -i.bak "s~^POSTGRES_PORT=.*~POSTGRES_PORT=5434~" .env - else - echo "POSTGRES_PORT=5434" >> .env - fi - - # Supabase Configuration (Only for full version) - if [ "$DEPLOYMENT_VERSION" = "full" ]; then - if [ -n "$SUPABASE_KEY" ]; then - if grep -q "^SUPABASE_KEY=" .env; then - sed -i.bak "s~^SUPABASE_KEY=.*~SUPABASE_KEY=$SUPABASE_KEY~" .env - else - echo "" >> .env - echo "# Supabase Keys" >> .env - echo "SUPABASE_KEY=$SUPABASE_KEY" >> .env - fi - fi - - if [ -n "$SERVICE_ROLE_KEY" ]; then - if grep -q "^SERVICE_ROLE_KEY=" .env; then - sed -i.bak "s~^SERVICE_ROLE_KEY=.*~SERVICE_ROLE_KEY=$SERVICE_ROLE_KEY~" .env - else - echo "SERVICE_ROLE_KEY=$SERVICE_ROLE_KEY" >> .env - fi - fi - - # Additional Supabase configuration - if grep -q "^SUPABASE_URL=" .env; then - sed -i.bak "s~^SUPABASE_URL=.*~SUPABASE_URL=http://localhost:8000~" .env - else - echo "SUPABASE_URL=http://localhost:8000" >> .env - fi - - if grep -q "^API_EXTERNAL_URL=" .env; then - sed -i.bak "s~^API_EXTERNAL_URL=.*~API_EXTERNAL_URL=http://localhost:8000~" .env - else - echo "API_EXTERNAL_URL=http://localhost:8000" >> .env - fi - - if grep -q "^SITE_URL=" .env; then - sed -i.bak "s~^SITE_URL=.*~SITE_URL=http://localhost:3011~" .env - else - echo "SITE_URL=http://localhost:3011" >> .env - fi - fi - - # Remove backup file - rm -f .env.bak - - echo " ✅ docker/.env updated successfully with localhost development URLs" -} - -# Function to show summary -show_summary() { - echo "🎉 Environment generation completed!" - - echo "" - echo "--------------------------------" - echo "" - - echo "🔣 Generated keys:" - echo " 🔑 MINIO_ACCESS_KEY: $MINIO_ACCESS_KEY" - echo " 🔑 MINIO_SECRET_KEY: $MINIO_SECRET_KEY" - if [ -n "$ELASTICSEARCH_API_KEY" ]; then - echo " 🔑 ELASTICSEARCH_API_KEY: $ELASTICSEARCH_API_KEY" - else - echo " ⚠️ ELASTICSEARCH_API_KEY: Not generated (Elasticsearch not available)" - fi - if [ -n "$SUPABASE_KEY" ]; then - echo " 🔑 SUPABASE_KEY: $SUPABASE_KEY" - fi - if [ -n "$SERVICE_ROLE_KEY" ]; then - echo " 🔑 SERVICE_ROLE_KEY: $SERVICE_ROLE_KEY" - fi - if [ -n "$SSH_USERNAME" ]; then - echo " 👤 SSH_USERNAME: $SSH_USERNAME" - fi - if [ -n "$SSH_PASSWORD" ]; then - echo " 🔑 SSH_PASSWORD: [HIDDEN]" - fi - if [ -z "$ELASTICSEARCH_API_KEY" ]; then - echo " ⚠️ Note: To generate ELASTICSEARCH_API_KEY later, please:" - echo " 1. Start Elasticsearch: docker-compose -p nexent up -d nexent-elasticsearch" - echo " 2. Wait for it to become healthy" - echo " 3. Run this script again or manually generate the API key" - fi -} - -# Main execution -main() { - # Step 1: Prepare .env file - prepare_env_file || { echo "❌ Failed to prepare .env file"; exit 1; } - - # Step 2: Update .env file - echo "" - update_env_file || { echo "❌ Failed to update .env file"; exit 1; } - - # Step 3: Show summary - show_summary -} - -# Run main function -main "$@" diff --git a/docker/init.sql b/docker/init.sql deleted file mode 100644 index 046bdecf1..000000000 --- a/docker/init.sql +++ /dev/null @@ -1,1939 +0,0 @@ --- 1. Create custom Schema (if not exists) -CREATE SCHEMA IF NOT EXISTS nexent; - --- 2. Switch to the Schema (subsequent operations default to this Schema) -SET search_path TO nexent; - -CREATE TABLE IF NOT EXISTS "conversation_message_t" ( - "message_id" SERIAL, - "conversation_id" int4, - "message_index" int4, - "message_role" varchar(30) COLLATE "pg_catalog"."default", - "message_content" varchar COLLATE "pg_catalog"."default", - "minio_files" varchar, - "opinion_flag" varchar(1), - "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, - "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "created_by" varchar(100) COLLATE "pg_catalog"."default", - "updated_by" varchar(100) COLLATE "pg_catalog"."default", - CONSTRAINT "conversation_message_t_pk" PRIMARY KEY ("message_id") -); -ALTER TABLE "conversation_message_t" OWNER TO "root"; -COMMENT ON COLUMN "conversation_message_t"."conversation_id" IS 'Formal foreign key, used to associate with the conversation'; -COMMENT ON COLUMN "conversation_message_t"."message_index" IS 'Sequence number, used for frontend display sorting'; -COMMENT ON COLUMN "conversation_message_t"."message_role" IS 'Role sending the message, such as system, assistant, user'; -COMMENT ON COLUMN "conversation_message_t"."message_content" IS 'Complete content of the message'; -COMMENT ON COLUMN "conversation_message_t"."minio_files" IS 'Images or documents uploaded by users in the chat interface, stored as a list'; -COMMENT ON COLUMN "conversation_message_t"."opinion_flag" IS 'User feedback on the conversation, enum value Y represents positive, N represents negative'; -COMMENT ON COLUMN "conversation_message_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; -COMMENT ON COLUMN "conversation_message_t"."create_time" IS 'Creation time, audit field'; -COMMENT ON COLUMN "conversation_message_t"."update_time" IS 'Update time, audit field'; -COMMENT ON COLUMN "conversation_message_t"."created_by" IS 'Creator ID, audit field'; -COMMENT ON COLUMN "conversation_message_t"."updated_by" IS 'Last updater ID, audit field'; -COMMENT ON TABLE "conversation_message_t" IS 'Carries specific response message content in conversations'; - -CREATE TABLE IF NOT EXISTS "conversation_message_unit_t" ( - "unit_id" SERIAL, - "message_id" int4, - "conversation_id" int4, - "unit_index" int4, - "unit_type" varchar(100) COLLATE "pg_catalog"."default", - "unit_content" varchar COLLATE "pg_catalog"."default", - "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, - "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "updated_by" varchar(100) COLLATE "pg_catalog"."default", - "created_by" varchar(100) COLLATE "pg_catalog"."default", - CONSTRAINT "conversation_message_unit_t_pk" PRIMARY KEY ("unit_id") -); -ALTER TABLE "conversation_message_unit_t" OWNER TO "root"; -COMMENT ON COLUMN "conversation_message_unit_t"."message_id" IS 'Formal foreign key, used to associate with the message'; -COMMENT ON COLUMN "conversation_message_unit_t"."conversation_id" IS 'Formal foreign key, used to associate with the conversation'; -COMMENT ON COLUMN "conversation_message_unit_t"."unit_index" IS 'Sequence number, used for frontend display sorting'; -COMMENT ON COLUMN "conversation_message_unit_t"."unit_type" IS 'Type of minimum response unit'; -COMMENT ON COLUMN "conversation_message_unit_t"."unit_content" IS 'Complete content of the minimum response unit'; -COMMENT ON COLUMN "conversation_message_unit_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; -COMMENT ON COLUMN "conversation_message_unit_t"."create_time" IS 'Creation time, audit field'; -COMMENT ON COLUMN "conversation_message_unit_t"."update_time" IS 'Update time, audit field'; -COMMENT ON COLUMN "conversation_message_unit_t"."updated_by" IS 'Last updater ID, audit field'; -COMMENT ON COLUMN "conversation_message_unit_t"."created_by" IS 'Creator ID, audit field'; -COMMENT ON TABLE "conversation_message_unit_t" IS 'Carries agent output content in each message'; - -CREATE TABLE IF NOT EXISTS "conversation_record_t" ( - "conversation_id" SERIAL, - "conversation_title" varchar(100) COLLATE "pg_catalog"."default", - "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, - "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "updated_by" varchar(100) COLLATE "pg_catalog"."default", - "created_by" varchar(100) COLLATE "pg_catalog"."default", - CONSTRAINT "conversation_record_t_pk" PRIMARY KEY ("conversation_id") -); -ALTER TABLE "conversation_record_t" OWNER TO "root"; -COMMENT ON COLUMN "conversation_record_t"."conversation_title" IS 'Conversation title'; -COMMENT ON COLUMN "conversation_record_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; -COMMENT ON COLUMN "conversation_record_t"."update_time" IS 'Update time, audit field'; -COMMENT ON COLUMN "conversation_record_t"."create_time" IS 'Creation time, audit field'; -COMMENT ON COLUMN "conversation_record_t"."updated_by" IS 'Last updater ID, audit field'; -COMMENT ON COLUMN "conversation_record_t"."created_by" IS 'Creator ID, audit field'; -COMMENT ON TABLE "conversation_record_t" IS 'Overall information of Q&A conversations'; - -CREATE TABLE IF NOT EXISTS "conversation_source_image_t" ( - "image_id" SERIAL, - "conversation_id" int4, - "message_id" int4, - "unit_id" int4, - "image_url" varchar COLLATE "pg_catalog"."default", - "cite_index" int4, - "search_type" varchar(100) COLLATE "pg_catalog"."default", - "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, - "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "created_by" varchar(100) COLLATE "pg_catalog"."default", - "updated_by" varchar(100) COLLATE "pg_catalog"."default", - CONSTRAINT "conversation_source_image_t_pk" PRIMARY KEY ("image_id") -); -ALTER TABLE "conversation_source_image_t" OWNER TO "root"; -COMMENT ON COLUMN "conversation_source_image_t"."conversation_id" IS 'Formal foreign key, used to associate with the conversation of the search source'; -COMMENT ON COLUMN "conversation_source_image_t"."message_id" IS 'Formal foreign key, used to associate with the conversation message of the search source'; -COMMENT ON COLUMN "conversation_source_image_t"."unit_id" IS 'Formal foreign key, used to associate with the minimum message unit of the search source (if any)'; -COMMENT ON COLUMN "conversation_source_image_t"."image_url" IS 'URL address of the image'; -COMMENT ON COLUMN "conversation_source_image_t"."cite_index" IS '[Reserved] Citation sequence number, used for precise tracing'; -COMMENT ON COLUMN "conversation_source_image_t"."search_type" IS '[Reserved] Search source type, used to distinguish the search tool used for this record, optional values web/local'; -COMMENT ON COLUMN "conversation_source_image_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; -COMMENT ON COLUMN "conversation_source_image_t"."create_time" IS 'Creation time, audit field'; -COMMENT ON COLUMN "conversation_source_image_t"."update_time" IS 'Update time, audit field'; -COMMENT ON COLUMN "conversation_source_image_t"."created_by" IS 'Creator ID, audit field'; -COMMENT ON COLUMN "conversation_source_image_t"."updated_by" IS 'Last updater ID, audit field'; -COMMENT ON TABLE "conversation_source_image_t" IS 'Carries search image source information for conversation messages'; - -CREATE TABLE IF NOT EXISTS "conversation_source_search_t" ( - "search_id" SERIAL, - "unit_id" int4, - "message_id" int4, - "conversation_id" int4, - "source_type" varchar(100) COLLATE "pg_catalog"."default", - "source_title" varchar(400) COLLATE "pg_catalog"."default", - "source_location" varchar(400) COLLATE "pg_catalog"."default", - "source_content" varchar COLLATE "pg_catalog"."default", - "score_overall" numeric(7,6), - "score_accuracy" numeric(7,6), - "score_semantic" numeric(7,6), - "published_date" timestamp(0), - "cite_index" int4, - "search_type" varchar(100) COLLATE "pg_catalog"."default", - "tool_sign" varchar(30) COLLATE "pg_catalog"."default", - "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, - "updated_by" varchar(100) COLLATE "pg_catalog"."default", - "created_by" varchar(100) COLLATE "pg_catalog"."default", - CONSTRAINT "conversation_source_search_t_pk" PRIMARY KEY ("search_id") -); -ALTER TABLE "conversation_source_search_t" OWNER TO "root"; -COMMENT ON COLUMN "conversation_source_search_t"."unit_id" IS 'Formal foreign key, used to associate with the minimum message unit of the search source (if any)'; -COMMENT ON COLUMN "conversation_source_search_t"."message_id" IS 'Formal foreign key, used to associate with the conversation message of the search source'; -COMMENT ON COLUMN "conversation_source_search_t"."conversation_id" IS 'Formal foreign key, used to associate with the conversation of the search source'; -COMMENT ON COLUMN "conversation_source_search_t"."source_type" IS 'Source type, used to distinguish if source_location is URL or path, optional values url/text'; -COMMENT ON COLUMN "conversation_source_search_t"."source_title" IS 'Title or filename of the search source'; -COMMENT ON COLUMN "conversation_source_search_t"."source_location" IS 'URL link or file path of the search source'; -COMMENT ON COLUMN "conversation_source_search_t"."source_content" IS 'Original text of the search source'; -COMMENT ON COLUMN "conversation_source_search_t"."score_overall" IS 'Overall similarity score between source and user query, calculated as weighted average of details'; -COMMENT ON COLUMN "conversation_source_search_t"."score_accuracy" IS 'Accuracy score'; -COMMENT ON COLUMN "conversation_source_search_t"."score_semantic" IS 'Semantic similarity score'; -COMMENT ON COLUMN "conversation_source_search_t"."published_date" IS 'Upload date of local file or network search date'; -COMMENT ON COLUMN "conversation_source_search_t"."cite_index" IS 'Citation sequence number, used for precise tracing'; -COMMENT ON COLUMN "conversation_source_search_t"."search_type" IS 'Search source type, specifically describes the search tool used for this record, optional values web_search/knowledge_base_search'; -COMMENT ON COLUMN "conversation_source_search_t"."tool_sign" IS 'Simple tool identifier, used to distinguish index sources in large model output summary text'; -COMMENT ON COLUMN "conversation_source_search_t"."create_time" IS 'Creation time, audit field'; -COMMENT ON COLUMN "conversation_source_search_t"."update_time" IS 'Update time, audit field'; -COMMENT ON COLUMN "conversation_source_search_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; -COMMENT ON COLUMN "conversation_source_search_t"."updated_by" IS 'Last updater ID, audit field'; -COMMENT ON COLUMN "conversation_source_search_t"."created_by" IS 'Creator ID, audit field'; -COMMENT ON TABLE "conversation_source_search_t" IS 'Carries search text source information referenced in conversation response messages'; - -CREATE TABLE IF NOT EXISTS "model_record_t" ( - "model_id" SERIAL, - "model_repo" varchar(100) COLLATE "pg_catalog"."default", - "model_name" varchar(100) COLLATE "pg_catalog"."default" NOT NULL, - "model_factory" varchar(100) COLLATE "pg_catalog"."default", - "model_type" varchar(100) COLLATE "pg_catalog"."default", - "api_key" varchar(500) COLLATE "pg_catalog"."default", - "base_url" varchar(500) COLLATE "pg_catalog"."default", - "max_tokens" int4, - "used_token" int4, - "expected_chunk_size" int4, - "maximum_chunk_size" int4, - "chunk_batch" int4, - "display_name" varchar(100) COLLATE "pg_catalog"."default", - "connect_status" varchar(100) COLLATE "pg_catalog"."default", - "ssl_verify" boolean DEFAULT true, - "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, - "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "updated_by" varchar(100) COLLATE "pg_catalog"."default", - "created_by" varchar(100) COLLATE "pg_catalog"."default", - "tenant_id" varchar(100) COLLATE "pg_catalog"."default" DEFAULT 'tenant_id', - "model_appid" varchar(100) COLLATE "pg_catalog"."default" DEFAULT '', - "access_token" varchar(100) COLLATE "pg_catalog"."default" DEFAULT '', - "concurrency_limit" INTEGER DEFAULT NULL, - "timeout_seconds" INTEGER DEFAULT 120, - CONSTRAINT "nexent_models_t_pk" PRIMARY KEY ("model_id") -); -ALTER TABLE "model_record_t" OWNER TO "root"; -COMMENT ON COLUMN "model_record_t"."model_id" IS 'Model ID, unique primary key'; -COMMENT ON COLUMN "model_record_t"."model_repo" IS 'Model path address'; -COMMENT ON COLUMN "model_record_t"."model_name" IS 'Model name'; -COMMENT ON COLUMN "model_record_t"."model_factory" IS 'Model manufacturer, determines specific format of api-key and model response. Currently defaults to OpenAI-API-Compatible'; -COMMENT ON COLUMN "model_record_t"."model_type" IS 'Model type, e.g. chat, embedding, rerank, tts, asr'; -COMMENT ON COLUMN "model_record_t"."api_key" IS 'Model API key, used for authentication for some models'; -COMMENT ON COLUMN "model_record_t"."base_url" IS 'Base URL address, used for requesting remote model services'; -COMMENT ON COLUMN "model_record_t"."max_tokens" IS 'Maximum available tokens for the model'; -COMMENT ON COLUMN "model_record_t"."used_token" IS 'Number of tokens already used by the model in Q&A'; -COMMENT ON COLUMN "model_record_t".expected_chunk_size IS 'Expected chunk size for embedding models, used during document chunking'; -COMMENT ON COLUMN "model_record_t".maximum_chunk_size IS 'Maximum chunk size for embedding models, used during document chunking'; -COMMENT ON COLUMN "model_record_t"."display_name" IS 'Model name displayed directly in frontend, customized by user'; -COMMENT ON COLUMN "model_record_t"."connect_status" IS 'Model connectivity status from last check, optional values: "检测中"、"可用"、"不可用"'; -COMMENT ON COLUMN "model_record_t"."ssl_verify" IS 'Whether to verify SSL certificates when connecting to this model API. Default is true. Set to false for local services without SSL support.'; -COMMENT ON COLUMN "model_record_t"."create_time" IS 'Creation time, audit field'; -COMMENT ON COLUMN "model_record_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; -COMMENT ON COLUMN "model_record_t"."update_time" IS 'Update time, audit field'; -COMMENT ON COLUMN "model_record_t"."updated_by" IS 'Last updater ID, audit field'; -COMMENT ON COLUMN "model_record_t"."created_by" IS 'Creator ID, audit field'; -COMMENT ON COLUMN "model_record_t"."tenant_id" IS 'Tenant ID for filtering'; -COMMENT ON COLUMN "model_record_t"."model_appid" IS 'Application ID for model authentication.'; -COMMENT ON COLUMN "model_record_t"."access_token" IS 'Access token for model authentication.'; -COMMENT ON COLUMN "model_record_t"."concurrency_limit" IS 'Maximum concurrent requests for this model. Default is NULL (unlimited).'; -COMMENT ON COLUMN "model_record_t"."timeout_seconds" IS 'Request timeout in seconds for this model. Default is 120 seconds.'; -COMMENT ON TABLE "model_record_t" IS 'List of models defined by users in the configuration page'; - -INSERT INTO "nexent"."model_record_t" ("model_repo", "model_name", "model_factory", "model_type", "api_key", "base_url", "max_tokens", "used_token", "display_name", "connect_status") VALUES ('', 'volcano_tts', 'OpenAI-API-Compatible', 'tts', '', '', 0, 0, 'volcano_tts', 'unavailable'); -INSERT INTO "nexent"."model_record_t" ("model_repo", "model_name", "model_factory", "model_type", "api_key", "base_url", "max_tokens", "used_token", "display_name", "connect_status") VALUES ('', 'volcano_stt', 'OpenAI-API-Compatible', 'stt', '', '', 0, 0, 'volcano_stt', 'unavailable'); - -CREATE TABLE IF NOT EXISTS "knowledge_record_t" ( - "knowledge_id" SERIAL, - "index_name" varchar(100) COLLATE "pg_catalog"."default", - "knowledge_name" varchar(100) COLLATE "pg_catalog"."default", - "knowledge_describe" varchar(3000) COLLATE "pg_catalog"."default", - "tenant_id" varchar(100) COLLATE "pg_catalog"."default", - "knowledge_sources" varchar(100) COLLATE "pg_catalog"."default", - "embedding_model_name" varchar(200) COLLATE "pg_catalog"."default", - "embedding_model_id" INTEGER, - "group_ids" varchar, - "ingroup_permission" varchar(30), - "create_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "update_time" timestamp(0) DEFAULT CURRENT_TIMESTAMP, - "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying, - "updated_by" varchar(100) COLLATE "pg_catalog"."default", - "created_by" varchar(100) COLLATE "pg_catalog"."default", - "summary_frequency" varchar(10) COLLATE "pg_catalog"."default", - "last_summary_time" timestamp(0), - "last_doc_update_time" timestamp(0), - "preserve_source_file" boolean NOT NULL DEFAULT true, - CONSTRAINT "knowledge_record_t_pk" PRIMARY KEY ("knowledge_id") -); -ALTER TABLE "knowledge_record_t" OWNER TO "root"; -COMMENT ON COLUMN "knowledge_record_t"."knowledge_id" IS 'Knowledge base ID, unique primary key'; -COMMENT ON COLUMN "knowledge_record_t"."index_name" IS 'Internal Elasticsearch index name'; -COMMENT ON COLUMN "knowledge_record_t"."knowledge_name" IS 'User-facing knowledge base name (display name), mapped to internal index_name'; -COMMENT ON COLUMN "knowledge_record_t"."knowledge_describe" IS 'Knowledge base description'; -COMMENT ON COLUMN "knowledge_record_t"."tenant_id" IS 'Tenant ID'; -COMMENT ON COLUMN "knowledge_record_t"."knowledge_sources" IS 'Knowledge base sources'; -COMMENT ON COLUMN "knowledge_record_t"."embedding_model_name" IS 'Embedding model name, used to record the embedding model used by the knowledge base'; -COMMENT ON COLUMN "knowledge_record_t"."embedding_model_id" IS 'Embedding model ID, foreign key reference to model_record_t.model_id'; -COMMENT ON COLUMN "knowledge_record_t"."group_ids" IS 'Knowledge base group IDs list'; -COMMENT ON COLUMN "knowledge_record_t"."ingroup_permission" IS 'In-group permission: EDIT, READ_ONLY, PRIVATE'; -COMMENT ON COLUMN "knowledge_record_t"."create_time" IS 'Creation time, audit field'; -COMMENT ON COLUMN "knowledge_record_t"."update_time" IS 'Update time, audit field'; -COMMENT ON COLUMN "knowledge_record_t"."delete_flag" IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; -COMMENT ON COLUMN "knowledge_record_t"."updated_by" IS 'User who last updated the record, audit field'; -COMMENT ON COLUMN "knowledge_record_t"."created_by" IS 'User who created the record, audit field'; -COMMENT ON COLUMN "knowledge_record_t"."summary_frequency" IS 'Auto-summary frequency: 1h, 3h, 6h, 1d, 1w, or NULL (disabled)'; -COMMENT ON COLUMN "knowledge_record_t"."last_summary_time" IS 'Timestamp of last summary generation'; -COMMENT ON COLUMN "knowledge_record_t"."last_doc_update_time" IS 'Timestamp of last document add/delete operation, used for auto-summary optimization to skip unnecessary summary regeneration'; -COMMENT ON COLUMN "knowledge_record_t"."preserve_source_file" IS 'Whether to preserve uploaded source documents after vectorization'; -COMMENT ON COLUMN "knowledge_record_t"."updated_by" IS 'Last updater ID, audit field'; -COMMENT ON COLUMN "knowledge_record_t"."created_by" IS 'Creator ID, audit field'; -COMMENT ON TABLE "knowledge_record_t" IS 'Records knowledge base description and status information'; - --- Create the ag_tool_info_t table -CREATE TABLE IF NOT EXISTS nexent.ag_tool_info_t ( - tool_id SERIAL PRIMARY KEY NOT NULL, - name VARCHAR(100), - origin_name VARCHAR(100), - class_name VARCHAR(100), - description VARCHAR, - source VARCHAR(100), - author VARCHAR(100), - usage VARCHAR(100), - params JSON, - inputs VARCHAR, - output_type VARCHAR(100), - category VARCHAR(100), - is_available BOOLEAN DEFAULT FALSE, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Trigger to update update_time when the record is modified -CREATE OR REPLACE FUNCTION update_ag_tool_info_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER update_ag_tool_info_update_time_trigger -BEFORE UPDATE ON nexent.ag_tool_info_t -FOR EACH ROW -EXECUTE FUNCTION update_ag_tool_info_update_time(); - --- Add comment to the table -COMMENT ON TABLE nexent.ag_tool_info_t IS 'Information table for prompt tools'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_tool_info_t.tool_id IS 'ID'; -COMMENT ON COLUMN nexent.ag_tool_info_t.name IS 'Unique key name'; -COMMENT ON COLUMN nexent.ag_tool_info_t.class_name IS 'Tool class name, used when the tool is instantiated'; -COMMENT ON COLUMN nexent.ag_tool_info_t.description IS 'Prompt tool description'; -COMMENT ON COLUMN nexent.ag_tool_info_t.source IS 'Source'; -COMMENT ON COLUMN nexent.ag_tool_info_t.author IS 'Tool author'; -COMMENT ON COLUMN nexent.ag_tool_info_t.usage IS 'Usage'; -COMMENT ON COLUMN nexent.ag_tool_info_t.params IS 'Tool parameter information (json)'; -COMMENT ON COLUMN nexent.ag_tool_info_t.inputs IS 'Prompt tool inputs description'; -COMMENT ON COLUMN nexent.ag_tool_info_t.output_type IS 'Prompt tool output description'; -COMMENT ON COLUMN nexent.ag_tool_info_t.is_available IS 'Whether the tool can be used under the current main service'; -COMMENT ON COLUMN nexent.ag_tool_info_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.ag_tool_info_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.ag_tool_info_t.created_by IS 'Creator'; -COMMENT ON COLUMN nexent.ag_tool_info_t.updated_by IS 'Updater'; -COMMENT ON COLUMN nexent.ag_tool_info_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create the ag_tenant_agent_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.ag_tenant_agent_t ( - agent_id SERIAL NOT NULL, - name VARCHAR(100), - display_name VARCHAR(100), - description VARCHAR, - business_description VARCHAR, - author VARCHAR(100), - model_name VARCHAR(100), - model_id INTEGER, - business_logic_model_name VARCHAR(100), - business_logic_model_id INTEGER, - prompt_template_id INTEGER, - prompt_template_name VARCHAR(100), - max_steps INTEGER, - duty_prompt TEXT, - constraint_prompt TEXT, - few_shots_prompt TEXT, - parent_agent_id INTEGER, - tenant_id VARCHAR(100), - group_ids VARCHAR, - enabled BOOLEAN DEFAULT FALSE, - is_new BOOLEAN DEFAULT FALSE, - provide_run_summary BOOLEAN DEFAULT FALSE, - enable_context_manager BOOLEAN DEFAULT FALSE, - verification_config JSONB, - version_no INTEGER DEFAULT 0 NOT NULL, - current_version_no INTEGER NULL, - ingroup_permission VARCHAR(30), - greeting_message TEXT, - example_questions JSONB, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N', - PRIMARY KEY (agent_id, version_no) -); - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_ag_tenant_agent_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Create a trigger to call the function before each update -CREATE TRIGGER update_ag_tenant_agent_update_time_trigger -BEFORE UPDATE ON nexent.ag_tenant_agent_t -FOR EACH ROW -EXECUTE FUNCTION update_ag_tenant_agent_update_time(); --- Add comments to the table -COMMENT ON TABLE nexent.ag_tenant_agent_t IS 'Information table for agents'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_tenant_agent_t.agent_id IS 'ID'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.name IS 'Agent name'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.display_name IS 'Agent display name'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.description IS 'Description'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.author IS 'Agent author'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.business_description IS 'Manually entered by the user to describe the entire business process'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.model_name IS '[DEPRECATED] Name of the model used, use model_id instead'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.model_id IS 'Model ID, foreign key reference to model_record_t.model_id'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.business_logic_model_name IS 'Model name used for business logic prompt generation'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.business_logic_model_id IS 'Model ID used for business logic prompt generation, foreign key reference to model_record_t.model_id'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.prompt_template_id IS 'Prompt template ID used for business logic prompt generation'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.prompt_template_name IS 'Prompt template name used for business logic prompt generation'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.max_steps IS 'Maximum number of steps'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.duty_prompt IS 'Duty prompt'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.constraint_prompt IS 'Constraint prompt'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.few_shots_prompt IS 'Few-shots prompt'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.parent_agent_id IS 'Parent Agent ID'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.tenant_id IS 'Belonging tenant'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.group_ids IS 'Agent group IDs list'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.enabled IS 'Enable flag'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.provide_run_summary IS 'Whether to provide the running summary to the manager agent'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.created_by IS 'Creator'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.updated_by IS 'Updater'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.is_new IS 'Whether this agent is marked as new for the user'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.current_version_no IS 'Current published version number. NULL means no version published yet'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.ingroup_permission IS 'In-group permission: EDIT, READ_ONLY, PRIVATE'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.enable_context_manager IS 'Whether to enable context management (compression) for this agent'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.verification_config IS 'Layered ReAct self-verification configuration'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.greeting_message IS 'Agent greeting message displayed on chat initial screen'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.example_questions IS 'List of example questions for starting a conversation with this agent'; - --- Create index for is_new queries -CREATE INDEX IF NOT EXISTS idx_ag_tenant_agent_t_is_new -ON nexent.ag_tenant_agent_t (tenant_id, is_new) -WHERE delete_flag = 'N'; - -CREATE TABLE IF NOT EXISTS nexent.ag_prompt_template_t ( - template_id SERIAL PRIMARY KEY, - template_name VARCHAR(100) NOT NULL, - description VARCHAR(500), - template_type VARCHAR(50) NOT NULL DEFAULT 'agent_generate', - tenant_id VARCHAR(100) NOT NULL, - user_id VARCHAR(100) NOT NULL, - template_content_zh JSONB NOT NULL, - template_content_en JSONB, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.ag_prompt_template_t OWNER TO "root"; - -CREATE OR REPLACE FUNCTION update_ag_prompt_template_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER update_ag_prompt_template_update_time_trigger -BEFORE UPDATE ON nexent.ag_prompt_template_t -FOR EACH ROW -EXECUTE FUNCTION update_ag_prompt_template_update_time(); - -COMMENT ON TABLE nexent.ag_prompt_template_t IS 'Prompt template table for user-defined business logic generation prompts'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.template_id IS 'Prompt template ID'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.template_name IS 'Prompt template name'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.description IS 'Prompt template description'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.template_type IS 'Prompt template type'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.template_content_zh IS 'Chinese prompt template content'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.template_content_en IS 'English prompt template content'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.created_by IS 'Creator'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.updated_by IS 'Updater'; -COMMENT ON COLUMN nexent.ag_prompt_template_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - -CREATE UNIQUE INDEX IF NOT EXISTS uq_prompt_template_user_name_active -ON nexent.ag_prompt_template_t (tenant_id, user_id, template_name) -WHERE delete_flag = 'N'; - -CREATE INDEX IF NOT EXISTS idx_ag_prompt_template_t_user -ON nexent.ag_prompt_template_t (tenant_id, user_id, template_type) -WHERE delete_flag = 'N'; - -INSERT INTO nexent.ag_prompt_template_t ( - template_id, - template_name, - description, - template_type, - tenant_id, - user_id, - template_content_zh, - template_content_en, - created_by, - updated_by, - delete_flag -) -VALUES ( - 0, - 'system_default', - 'System default prompt template', - 'agent_generate', - 'tenant_id', - 'user_id', - '{}'::jsonb, - '{}'::jsonb, - 'user_id', - 'user_id', - 'N' -) -ON CONFLICT (template_id) DO UPDATE SET - template_name = EXCLUDED.template_name, - description = EXCLUDED.description, - template_type = EXCLUDED.template_type, - tenant_id = EXCLUDED.tenant_id, - user_id = EXCLUDED.user_id, - template_content_zh = EXCLUDED.template_content_zh, - template_content_en = EXCLUDED.template_content_en, - updated_by = EXCLUDED.updated_by, - delete_flag = 'N'; - - --- Create the ag_tool_instance_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.ag_tool_instance_t ( - tool_instance_id SERIAL NOT NULL, - tool_id INTEGER, - agent_id INTEGER, - params JSON, - user_id VARCHAR(100), - tenant_id VARCHAR(100), - enabled BOOLEAN DEFAULT FALSE, - version_no INTEGER DEFAULT 0 NOT NULL, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N', - PRIMARY KEY (tool_instance_id, version_no) -); - --- Add comment to the table -COMMENT ON TABLE nexent.ag_tool_instance_t IS 'Information table for tenant tool configuration.'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_tool_instance_t.tool_instance_id IS 'ID'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.tool_id IS 'Tenant tool ID'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.agent_id IS 'Agent ID'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.params IS 'Parameter configuration'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.enabled IS 'Enable flag'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.update_time IS 'Update time'; - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_ag_tool_instance_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Add comment to the function -COMMENT ON FUNCTION update_ag_tool_instance_update_time() IS 'Function to update the update_time column when a record in ag_tool_instance_t is updated'; - --- Create a trigger to call the function before each update -CREATE TRIGGER update_ag_tool_instance_update_time_trigger -BEFORE UPDATE ON nexent.ag_tool_instance_t -FOR EACH ROW -EXECUTE FUNCTION update_ag_tool_instance_update_time(); - --- Add comment to the trigger -COMMENT ON TRIGGER update_ag_tool_instance_update_time_trigger ON nexent.ag_tool_instance_t IS 'Trigger to call update_ag_tool_instance_update_time function before each update on ag_tool_instance_t table'; - --- Create the tenant_config_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.tenant_config_t ( - tenant_config_id SERIAL PRIMARY KEY NOT NULL, - tenant_id VARCHAR(100), - user_id VARCHAR(100), - value_type VARCHAR(100), - config_key VARCHAR(100), - config_value TEXT, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Add comment to the table -COMMENT ON TABLE nexent.tenant_config_t IS 'Tenant configuration information table'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.tenant_config_t.tenant_config_id IS 'ID'; -COMMENT ON COLUMN nexent.tenant_config_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.tenant_config_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.tenant_config_t.value_type IS 'Value type'; -COMMENT ON COLUMN nexent.tenant_config_t.config_key IS 'Config key'; -COMMENT ON COLUMN nexent.tenant_config_t.config_value IS 'Config value'; -COMMENT ON COLUMN nexent.tenant_config_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.tenant_config_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.tenant_config_t.created_by IS 'Creator'; -COMMENT ON COLUMN nexent.tenant_config_t.updated_by IS 'Updater'; -COMMENT ON COLUMN nexent.tenant_config_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_tenant_config_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Create a trigger to call the function before each update -CREATE TRIGGER update_tenant_config_update_time_trigger -BEFORE UPDATE ON nexent.tenant_config_t -FOR EACH ROW -EXECUTE FUNCTION update_tenant_config_update_time(); - --- Create the mcp_record_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.mcp_record_t ( - mcp_id SERIAL PRIMARY KEY NOT NULL, - tenant_id VARCHAR(100), - user_id VARCHAR(100), - mcp_name VARCHAR(100), - mcp_server VARCHAR(500), - status BOOLEAN DEFAULT NULL, - container_id VARCHAR(200) DEFAULT NULL, - authorization_token VARCHAR(500) DEFAULT NULL, - custom_headers JSON DEFAULT NULL, - source VARCHAR(30), - registry_json JSONB, - config_json JSON, - enabled BOOLEAN DEFAULT TRUE, - tags TEXT[], - description TEXT, - container_port INTEGER, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); -ALTER TABLE "mcp_record_t" OWNER TO "root"; --- Add comment to the table -COMMENT ON TABLE nexent.mcp_record_t IS 'MCP (Model Context Protocol) records table'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.mcp_record_t.mcp_id IS 'MCP record ID, unique primary key'; -COMMENT ON COLUMN nexent.mcp_record_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.mcp_record_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.mcp_record_t.mcp_name IS 'MCP name'; -COMMENT ON COLUMN nexent.mcp_record_t.mcp_server IS 'MCP server address'; -COMMENT ON COLUMN nexent.mcp_record_t.status IS 'MCP server connection status, true=connected, false=disconnected, null=unknown'; -COMMENT ON COLUMN nexent.mcp_record_t.container_id IS 'Docker container ID for MCP service, NULL for non-containerized MCP'; -COMMENT ON COLUMN nexent.mcp_record_t.authorization_token IS 'Authorization token for MCP server authentication (e.g., Bearer token)'; -COMMENT ON COLUMN nexent.mcp_record_t.custom_headers IS 'Custom HTTP headers as JSON object for MCP server requests'; -COMMENT ON COLUMN nexent.mcp_record_t.create_time IS 'Creation time, audit field'; -COMMENT ON COLUMN nexent.mcp_record_t.update_time IS 'Update time, audit field'; -COMMENT ON COLUMN nexent.mcp_record_t.created_by IS 'Creator ID, audit field'; -COMMENT ON COLUMN nexent.mcp_record_t.updated_by IS 'Last updater ID, audit field'; -COMMENT ON COLUMN nexent.mcp_record_t.delete_flag IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; -COMMENT ON COLUMN nexent.mcp_record_t.source IS 'Source type: local/mcp_registry/community'; -COMMENT ON COLUMN nexent.mcp_record_t.registry_json IS 'Full MCP registry server.json snapshot'; -COMMENT ON COLUMN nexent.mcp_record_t.config_json IS 'MCP config data'; -COMMENT ON COLUMN nexent.mcp_record_t.enabled IS 'Enabled'; -COMMENT ON COLUMN nexent.mcp_record_t.tags IS 'Tags'; -COMMENT ON COLUMN nexent.mcp_record_t.description IS 'Description'; -COMMENT ON COLUMN nexent.mcp_record_t.container_port IS 'Host port bound for containerized MCP service'; - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_mcp_record_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Add comment to the function -COMMENT ON FUNCTION update_mcp_record_update_time() IS 'Function to update the update_time column when a record in mcp_record_t is updated'; - --- Create a trigger to call the function before each update -CREATE TRIGGER update_mcp_record_update_time_trigger -BEFORE UPDATE ON nexent.mcp_record_t -FOR EACH ROW -EXECUTE FUNCTION update_mcp_record_update_time(); - --- Add comment to the trigger -COMMENT ON TRIGGER update_mcp_record_update_time_trigger ON nexent.mcp_record_t IS 'Trigger to call update_mcp_record_update_time function before each update on mcp_record_t table'; - --- Add indexes for common management queries -CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tenant_delete - ON nexent.mcp_record_t (tenant_id, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tenant_name - ON nexent.mcp_record_t (tenant_id, mcp_name, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tenant_server - ON nexent.mcp_record_t (tenant_id, mcp_server, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tags_gin - ON nexent.mcp_record_t USING GIN (tags); - --- Create user tenant relationship table -CREATE TABLE IF NOT EXISTS nexent.user_tenant_t ( - user_tenant_id SERIAL PRIMARY KEY, - user_id VARCHAR(100) NOT NULL, - tenant_id VARCHAR(100) NOT NULL, - user_role VARCHAR(30) DEFAULT 'USER', - user_email VARCHAR(255), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag CHAR(1) DEFAULT 'N', - UNIQUE(user_id, tenant_id) -); - --- Add comment -COMMENT ON TABLE nexent.user_tenant_t IS 'User tenant relationship table'; -COMMENT ON COLUMN nexent.user_tenant_t.user_tenant_id IS 'User tenant relationship ID, primary key'; -COMMENT ON COLUMN nexent.user_tenant_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.user_tenant_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.user_tenant_t.user_role IS 'User role: SUPER_ADMIN, ADMIN, DEV, USER'; -COMMENT ON COLUMN nexent.user_tenant_t.user_email IS 'User email address'; -COMMENT ON COLUMN nexent.user_tenant_t.create_time IS 'Create time'; -COMMENT ON COLUMN nexent.user_tenant_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.user_tenant_t.created_by IS 'Created by'; -COMMENT ON COLUMN nexent.user_tenant_t.updated_by IS 'Updated by'; -COMMENT ON COLUMN nexent.user_tenant_t.delete_flag IS 'Delete flag, Y/N'; - --- Create the ag_agent_relation_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.ag_agent_relation_t ( - relation_id SERIAL NOT NULL, - selected_agent_id INTEGER, - parent_agent_id INTEGER, - tenant_id VARCHAR(100), - version_no INTEGER DEFAULT 0 NOT NULL, - selected_agent_version_no INTEGER, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N', - PRIMARY KEY (relation_id, version_no) -); - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_ag_agent_relation_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Create a trigger to call the function before each update -CREATE TRIGGER update_ag_agent_relation_update_time_trigger -BEFORE UPDATE ON nexent.ag_agent_relation_t -FOR EACH ROW -EXECUTE FUNCTION update_ag_agent_relation_update_time(); - --- Add comment to the table -COMMENT ON TABLE nexent.ag_agent_relation_t IS 'Agent parent-child relationship table'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_agent_relation_t.relation_id IS 'Relationship ID, primary key'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.selected_agent_id IS 'Selected agent ID'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.parent_agent_id IS 'Parent agent ID'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.selected_agent_version_no IS 'Pinned version of selected_agent_id. NULL = use child current published version at runtime (legacy/draft).'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.create_time IS 'Creation time, audit field'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.update_time IS 'Update time, audit field'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.created_by IS 'Creator ID, audit field'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.updated_by IS 'Last updater ID, audit field'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.delete_flag IS 'Delete flag, set to Y for soft delete, optional values Y/N'; - --- Create user memory config table -CREATE TABLE IF NOT EXISTS "memory_user_config_t" ( - "config_id" SERIAL PRIMARY KEY NOT NULL, - "tenant_id" varchar(100) COLLATE "pg_catalog"."default", - "user_id" varchar(100) COLLATE "pg_catalog"."default", - "value_type" varchar(100) COLLATE "pg_catalog"."default", - "config_key" varchar(100) COLLATE "pg_catalog"."default", - "config_value" varchar(100) COLLATE "pg_catalog"."default", - "create_time" timestamp(6) DEFAULT CURRENT_TIMESTAMP, - "update_time" timestamp(6) DEFAULT CURRENT_TIMESTAMP, - "created_by" varchar(100) COLLATE "pg_catalog"."default", - "updated_by" varchar(100) COLLATE "pg_catalog"."default", - "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N' -); - -COMMENT ON COLUMN "nexent"."memory_user_config_t"."config_id" IS 'ID'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."tenant_id" IS 'Tenant ID'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."user_id" IS 'User ID'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."value_type" IS 'Value type. Optional values: single/multi'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."config_key" IS 'Config key'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."config_value" IS 'Config value'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."create_time" IS 'Creation time'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."update_time" IS 'Update time'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."created_by" IS 'Creator'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."updated_by" IS 'Updater'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."delete_flag" IS 'Whether it is deleted. Optional values: Y/N'; - -COMMENT ON TABLE "nexent"."memory_user_config_t" IS 'User configuration of memory setting table'; - -CREATE OR REPLACE FUNCTION "update_memory_user_config_update_time"() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER "update_memory_user_config_update_time_trigger" -BEFORE UPDATE ON "nexent"."memory_user_config_t" -FOR EACH ROW -EXECUTE FUNCTION "update_memory_user_config_update_time"(); - - --- 1. Create tenant_invitation_code_t table for invitation codes -CREATE TABLE IF NOT EXISTS nexent.tenant_invitation_code_t ( - invitation_id SERIAL PRIMARY KEY, - tenant_id VARCHAR(100) NOT NULL, - invitation_code VARCHAR(100) NOT NULL, - group_ids VARCHAR, -- int4 list - capacity INT4 NOT NULL DEFAULT 1, - expiry_date TIMESTAMP(6) WITHOUT TIME ZONE, - status VARCHAR(30) NOT NULL, - code_type VARCHAR(30) NOT NULL, - create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Add comments for tenant_invitation_code_t table -COMMENT ON TABLE nexent.tenant_invitation_code_t IS 'Tenant invitation code information table'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.invitation_id IS 'Invitation ID, primary key'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.tenant_id IS 'Tenant ID, foreign key'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.invitation_code IS 'Invitation code'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.group_ids IS 'Associated group IDs list'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.capacity IS 'Invitation code capacity'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.expiry_date IS 'Invitation code expiry date'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.status IS 'Invitation code status: IN_USE, EXPIRE, DISABLE, RUN_OUT'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.code_type IS 'Invitation code type: ADMIN_INVITE, DEV_INVITE, USER_INVITE, ASSET_OWNER_INVITE'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.create_time IS 'Create time'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.created_by IS 'Created by'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.updated_by IS 'Updated by'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.delete_flag IS 'Delete flag, Y/N'; - --- 2. Create tenant_invitation_record_t table for invitation usage records -CREATE TABLE IF NOT EXISTS nexent.tenant_invitation_record_t ( - invitation_record_id SERIAL PRIMARY KEY, - invitation_id INT4 NOT NULL, - user_id VARCHAR(100) NOT NULL, - create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Add comments for tenant_invitation_record_t table -COMMENT ON TABLE nexent.tenant_invitation_record_t IS 'Tenant invitation record table'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.invitation_record_id IS 'Invitation record ID, primary key'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.invitation_id IS 'Invitation ID, foreign key'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.create_time IS 'Create time'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.created_by IS 'Created by'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.updated_by IS 'Updated by'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.delete_flag IS 'Delete flag, Y/N'; - --- 3. Create tenant_group_info_t table for group information -CREATE TABLE IF NOT EXISTS nexent.tenant_group_info_t ( - group_id SERIAL PRIMARY KEY, - tenant_id VARCHAR(100) NOT NULL, - group_name VARCHAR(100) NOT NULL, - group_description VARCHAR(500), - create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Add comments for tenant_group_info_t table -COMMENT ON TABLE nexent.tenant_group_info_t IS 'Tenant group information table'; -COMMENT ON COLUMN nexent.tenant_group_info_t.group_id IS 'Group ID, primary key'; -COMMENT ON COLUMN nexent.tenant_group_info_t.tenant_id IS 'Tenant ID, foreign key'; -COMMENT ON COLUMN nexent.tenant_group_info_t.group_name IS 'Group name'; -COMMENT ON COLUMN nexent.tenant_group_info_t.group_description IS 'Group description'; -COMMENT ON COLUMN nexent.tenant_group_info_t.create_time IS 'Create time'; -COMMENT ON COLUMN nexent.tenant_group_info_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.tenant_group_info_t.created_by IS 'Created by'; -COMMENT ON COLUMN nexent.tenant_group_info_t.updated_by IS 'Updated by'; -COMMENT ON COLUMN nexent.tenant_group_info_t.delete_flag IS 'Delete flag, Y/N'; - --- 4. Create tenant_group_user_t table for group user membership -CREATE TABLE IF NOT EXISTS nexent.tenant_group_user_t ( - group_user_id SERIAL PRIMARY KEY, - group_id INT4 NOT NULL, - user_id VARCHAR(100) NOT NULL, - create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Add comments for tenant_group_user_t table -COMMENT ON TABLE nexent.tenant_group_user_t IS 'Tenant group user membership table'; -COMMENT ON COLUMN nexent.tenant_group_user_t.group_user_id IS 'Group user ID, primary key'; -COMMENT ON COLUMN nexent.tenant_group_user_t.group_id IS 'Group ID, foreign key'; -COMMENT ON COLUMN nexent.tenant_group_user_t.user_id IS 'User ID, foreign key'; -COMMENT ON COLUMN nexent.tenant_group_user_t.create_time IS 'Create time'; -COMMENT ON COLUMN nexent.tenant_group_user_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.tenant_group_user_t.created_by IS 'Created by'; -COMMENT ON COLUMN nexent.tenant_group_user_t.updated_by IS 'Updated by'; -COMMENT ON COLUMN nexent.tenant_group_user_t.delete_flag IS 'Delete flag, Y/N'; - --- 5. Create role_permission_t table for role permissions -CREATE TABLE IF NOT EXISTS nexent.role_permission_t ( - role_permission_id SERIAL PRIMARY KEY, - user_role VARCHAR(30) NOT NULL, - permission_category VARCHAR(30), - permission_type VARCHAR(30), - permission_subtype VARCHAR(30) -); - --- Add comments for role_permission_t table -COMMENT ON TABLE nexent.role_permission_t IS 'Role permission configuration table'; -COMMENT ON COLUMN nexent.role_permission_t.role_permission_id IS 'Role permission ID, primary key'; -COMMENT ON COLUMN nexent.role_permission_t.user_role IS 'User role: SU, ADMIN, DEV, USER'; -COMMENT ON COLUMN nexent.role_permission_t.permission_category IS 'Permission category'; -COMMENT ON COLUMN nexent.role_permission_t.permission_type IS 'Permission type'; -COMMENT ON COLUMN nexent.role_permission_t.permission_subtype IS 'Permission subtype'; - --- 6. Insert role permission data after clearing old data -DELETE FROM nexent.role_permission_t; - -INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES -(1, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(2, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(3, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/tenant-resources'), -(4, 'SU', 'RESOURCE', 'AGENT', 'READ'), -(5, 'SU', 'RESOURCE', 'AGENT', 'DELETE'), -(6, 'SU', 'RESOURCE', 'KB', 'READ'), -(7, 'SU', 'RESOURCE', 'KB', 'DELETE'), -(8, 'SU', 'RESOURCE', 'KB.GROUPS', 'READ'), -(9, 'SU', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(10, 'SU', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(11, 'SU', 'RESOURCE', 'USER.ROLE', 'READ'), -(12, 'SU', 'RESOURCE', 'USER.ROLE', 'UPDATE'), -(13, 'SU', 'RESOURCE', 'USER.ROLE', 'DELETE'), -(14, 'SU', 'RESOURCE', 'MCP', 'READ'), -(15, 'SU', 'RESOURCE', 'MCP', 'DELETE'), -(16, 'SU', 'RESOURCE', 'MEM.SETTING', 'READ'), -(17, 'SU', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(18, 'SU', 'RESOURCE', 'MEM.AGENT', 'READ'), -(19, 'SU', 'RESOURCE', 'MEM.AGENT', 'DELETE'), -(20, 'SU', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(21, 'SU', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(22, 'SU', 'RESOURCE', 'MODEL', 'CREATE'), -(23, 'SU', 'RESOURCE', 'MODEL', 'READ'), -(24, 'SU', 'RESOURCE', 'MODEL', 'UPDATE'), -(25, 'SU', 'RESOURCE', 'MODEL', 'DELETE'), -(26, 'SU', 'RESOURCE', 'TENANT', 'CREATE'), -(27, 'SU', 'RESOURCE', 'TENANT', 'READ'), -(28, 'SU', 'RESOURCE', 'TENANT', 'UPDATE'), -(29, 'SU', 'RESOURCE', 'TENANT', 'DELETE'), -(30, 'SU', 'RESOURCE', 'TENANT.LIST', 'READ'), -(31, 'SU', 'RESOURCE', 'TENANT.INFO', 'READ'), -(32, 'SU', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), -(33, 'SU', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), -(34, 'SU', 'RESOURCE', 'TENANT.INVITE', 'READ'), -(35, 'SU', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), -(36, 'SU', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), -(37, 'SU', 'RESOURCE', 'GROUP', 'CREATE'), -(38, 'SU', 'RESOURCE', 'GROUP', 'READ'), -(39, 'SU', 'RESOURCE', 'GROUP', 'UPDATE'), -(40, 'SU', 'RESOURCE', 'GROUP', 'DELETE'), -(41, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(42, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(43, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), -(44, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(45, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), -(46, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), -(47, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(48, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), -(49, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(50, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(51, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(52, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(53, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/tenant-resources'), -(54, 'ADMIN', 'RESOURCE', 'AGENT', 'CREATE'), -(55, 'ADMIN', 'RESOURCE', 'AGENT', 'READ'), -(56, 'ADMIN', 'RESOURCE', 'AGENT', 'UPDATE'), -(57, 'ADMIN', 'RESOURCE', 'AGENT', 'DELETE'), -(58, 'ADMIN', 'RESOURCE', 'KB', 'CREATE'), -(59, 'ADMIN', 'RESOURCE', 'KB', 'READ'), -(60, 'ADMIN', 'RESOURCE', 'KB', 'UPDATE'), -(61, 'ADMIN', 'RESOURCE', 'KB', 'DELETE'), -(62, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'READ'), -(63, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(64, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(65, 'ADMIN', 'RESOURCE', 'USER.ROLE', 'READ'), -(66, 'ADMIN', 'RESOURCE', 'MCP', 'CREATE'), -(67, 'ADMIN', 'RESOURCE', 'MCP', 'READ'), -(68, 'ADMIN', 'RESOURCE', 'MCP', 'UPDATE'), -(69, 'ADMIN', 'RESOURCE', 'MCP', 'DELETE'), -(70, 'ADMIN', 'RESOURCE', 'MEM.SETTING', 'READ'), -(71, 'ADMIN', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(72, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'CREATE'), -(73, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'READ'), -(74, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'DELETE'), -(75, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(76, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(77, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(78, 'ADMIN', 'RESOURCE', 'MODEL', 'CREATE'), -(79, 'ADMIN', 'RESOURCE', 'MODEL', 'READ'), -(80, 'ADMIN', 'RESOURCE', 'MODEL', 'UPDATE'), -(81, 'ADMIN', 'RESOURCE', 'MODEL', 'DELETE'), -(82, 'ADMIN', 'RESOURCE', 'TENANT.INFO', 'READ'), -(83, 'ADMIN', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), -(84, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), -(85, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'READ'), -(86, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), -(87, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), -(88, 'ADMIN', 'RESOURCE', 'GROUP', 'CREATE'), -(89, 'ADMIN', 'RESOURCE', 'GROUP', 'READ'), -(90, 'ADMIN', 'RESOURCE', 'GROUP', 'UPDATE'), -(91, 'ADMIN', 'RESOURCE', 'GROUP', 'DELETE'), -(92, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(93, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(94, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), -(95, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(96, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), -(97, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), -(98, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(99, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), -(100, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(101, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(102, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(103, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(104, 'DEV', 'RESOURCE', 'AGENT', 'CREATE'), -(105, 'DEV', 'RESOURCE', 'AGENT', 'READ'), -(106, 'DEV', 'RESOURCE', 'AGENT', 'UPDATE'), -(107, 'DEV', 'RESOURCE', 'AGENT', 'DELETE'), -(108, 'DEV', 'RESOURCE', 'KB', 'CREATE'), -(109, 'DEV', 'RESOURCE', 'KB', 'READ'), -(110, 'DEV', 'RESOURCE', 'KB', 'UPDATE'), -(111, 'DEV', 'RESOURCE', 'KB', 'DELETE'), -(112, 'DEV', 'RESOURCE', 'KB.GROUPS', 'READ'), -(113, 'DEV', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(114, 'DEV', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(115, 'DEV', 'RESOURCE', 'USER.ROLE', 'READ'), -(116, 'DEV', 'RESOURCE', 'MCP', 'CREATE'), -(117, 'DEV', 'RESOURCE', 'MCP', 'READ'), -(118, 'DEV', 'RESOURCE', 'MCP', 'UPDATE'), -(119, 'DEV', 'RESOURCE', 'MCP', 'DELETE'), -(120, 'DEV', 'RESOURCE', 'MEM.SETTING', 'READ'), -(121, 'DEV', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(122, 'DEV', 'RESOURCE', 'MEM.AGENT', 'READ'), -(123, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(124, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(125, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(126, 'DEV', 'RESOURCE', 'MODEL', 'READ'), -(127, 'DEV', 'RESOURCE', 'TENANT.INFO', 'READ'), -(128, 'DEV', 'RESOURCE', 'GROUP', 'READ'), -(129, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(130, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(131, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(132, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(133, 'USER', 'RESOURCE', 'AGENT', 'READ'), -(134, 'USER', 'RESOURCE', 'USER.ROLE', 'READ'), -(135, 'USER', 'RESOURCE', 'MEM.SETTING', 'READ'), -(136, 'USER', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(137, 'USER', 'RESOURCE', 'MEM.AGENT', 'READ'), -(138, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(139, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(140, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(141, 'USER', 'RESOURCE', 'TENANT.INFO', 'READ'), -(142, 'USER', 'RESOURCE', 'GROUP', 'READ'), -(143, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(144, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(145, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), -(146, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(147, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), -(148, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), -(149, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(150, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), -(151, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(152, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(153, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(154, 'SPEED', 'RESOURCE', 'AGENT', 'CREATE'), -(155, 'SPEED', 'RESOURCE', 'AGENT', 'READ'), -(156, 'SPEED', 'RESOURCE', 'AGENT', 'UPDATE'), -(157, 'SPEED', 'RESOURCE', 'AGENT', 'DELETE'), -(158, 'SPEED', 'RESOURCE', 'KB', 'CREATE'), -(159, 'SPEED', 'RESOURCE', 'KB', 'READ'), -(160, 'SPEED', 'RESOURCE', 'KB', 'UPDATE'), -(161, 'SPEED', 'RESOURCE', 'KB', 'DELETE'), -(166, 'SPEED', 'RESOURCE', 'MCP', 'CREATE'), -(167, 'SPEED', 'RESOURCE', 'MCP', 'READ'), -(168, 'SPEED', 'RESOURCE', 'MCP', 'UPDATE'), -(169, 'SPEED', 'RESOURCE', 'MCP', 'DELETE'), -(170, 'SPEED', 'RESOURCE', 'MEM.SETTING', 'READ'), -(171, 'SPEED', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(172, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'CREATE'), -(173, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'READ'), -(174, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'DELETE'), -(175, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(176, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(177, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(178, 'SPEED', 'RESOURCE', 'MODEL', 'CREATE'), -(179, 'SPEED', 'RESOURCE', 'MODEL', 'READ'), -(180, 'SPEED', 'RESOURCE', 'MODEL', 'UPDATE'), -(181, 'SPEED', 'RESOURCE', 'MODEL', 'DELETE'), -(182, 'SPEED', 'RESOURCE', 'TENANT.INFO', 'READ'), -(183, 'SPEED', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), -(184, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), -(185, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'READ'), -(186, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), -(187, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), -(188, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'CREATE'), -(189, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'READ'), -(190, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'UPDATE'), -(191, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'DELETE'), -(192, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(193, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), -(194, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(195, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(196, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(197, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), -(198, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(199, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'CREATE'), -(200, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'READ'), -(201, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'UPDATE'), -(202, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'DELETE'), -(203, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'CREATE'), -(204, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'READ'), -(205, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'UPDATE'), -(206, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'DELETE'), -(207, 'ASSET_OWNER', 'RESOURCE', 'KB', 'CREATE'), -(208, 'ASSET_OWNER', 'RESOURCE', 'KB', 'READ'), -(209, 'ASSET_OWNER', 'RESOURCE', 'KB', 'UPDATE'), -(210, 'ASSET_OWNER', 'RESOURCE', 'KB', 'DELETE'), -(211, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'CREATE'), -(212, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'READ'), -(213, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'UPDATE'), -(214, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'DELETE'), -(215, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'CREATE'), -(216, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'READ'), -(217, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'UPDATE'), -(218, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'DELETE'), -(219, 'ASSET_OWNER', 'RESOURCE', 'USER.ROLE', 'READ'), -(220, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(221, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/asset-owner-resources') -; - --- Insert SPEED role user into user_tenant_t table if not exists -INSERT INTO nexent.user_tenant_t (user_id, tenant_id, user_role, user_email, created_by, updated_by) -VALUES ('user_id', 'tenant_id', 'SPEED', '', 'system', 'system') -ON CONFLICT (user_id, tenant_id) DO NOTHING; - --- Create the ag_tenant_agent_version_t table for agent version management -CREATE TABLE IF NOT EXISTS nexent.ag_tenant_agent_version_t ( - id BIGSERIAL PRIMARY KEY, - tenant_id VARCHAR(100) NOT NULL, - agent_id INTEGER NOT NULL, - version_no INTEGER NOT NULL, - version_name VARCHAR(100), - release_note TEXT, - source_version_no INTEGER NULL, - source_type VARCHAR(30) NULL, - status VARCHAR(30) DEFAULT 'RELEASED', - is_a2a BOOLEAN DEFAULT FALSE, - created_by VARCHAR(100) NOT NULL, - create_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - updated_by VARCHAR(100), - update_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.ag_tenant_agent_version_t OWNER TO "root"; - --- Add comments for version fields in existing tables -COMMENT ON COLUMN nexent.ag_tenant_agent_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.current_version_no IS 'Current published version number. NULL means no version published yet'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; - --- Add comments for ag_tenant_agent_version_t table -COMMENT ON TABLE nexent.ag_tenant_agent_version_t IS 'Agent version metadata table. Stores version info, release notes, and version lineage.'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.id IS 'Primary key, auto-increment'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.agent_id IS 'Agent ID'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.version_no IS 'Version number, starts from 1. Does not include 0 (draft)'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.version_name IS 'User-defined version name for display (e.g., "Stable v2.1", "Hotfix-001"). NULL means use version_no as display.'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.release_note IS 'Release notes / publish remarks'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.source_version_no IS 'Source version number. If this version is a rollback, record the source version number.'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.source_type IS 'Source type: NORMAL (normal publish) / ROLLBACK (rollback and republish).'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.status IS 'Version status: RELEASED / DISABLED / ARCHIVED'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.is_a2a IS 'Whether this version is published as an A2A Server agent'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.created_by IS 'User who published this version'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.create_time IS 'Version creation timestamp'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.updated_by IS 'Last user who updated this version'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.update_time IS 'Last update timestamp'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.delete_flag IS 'Soft delete flag: Y/N'; - --- Create the user_token_info_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.user_token_info_t ( - token_id SERIAL4 PRIMARY KEY NOT NULL, - access_key VARCHAR(100) NOT NULL, - user_id VARCHAR(100) NOT NULL, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE "user_token_info_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.user_token_info_t IS 'User token (AK/SK) information table'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.user_token_info_t.token_id IS 'Token ID, unique primary key'; -COMMENT ON COLUMN nexent.user_token_info_t.access_key IS 'Access Key (AK)'; -COMMENT ON COLUMN nexent.user_token_info_t.user_id IS 'User ID who owns this token'; -COMMENT ON COLUMN nexent.user_token_info_t.create_time IS 'Creation time, audit field'; -COMMENT ON COLUMN nexent.user_token_info_t.update_time IS 'Update time, audit field'; -COMMENT ON COLUMN nexent.user_token_info_t.created_by IS 'Creator ID, audit field'; -COMMENT ON COLUMN nexent.user_token_info_t.updated_by IS 'Last updater ID, audit field'; -COMMENT ON COLUMN nexent.user_token_info_t.delete_flag IS 'Soft delete flag, Y means deleted'; - - --- Create the user_token_usage_log_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.user_token_usage_log_t ( - token_usage_id SERIAL4 PRIMARY KEY NOT NULL, - token_id INT4 NOT NULL, - call_function_name VARCHAR(100), - related_id INT4, - meta_data JSONB, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE "user_token_usage_log_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.user_token_usage_log_t IS 'User token usage log table'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.user_token_usage_log_t.token_usage_id IS 'Token usage log ID, unique primary key'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.token_id IS 'Foreign key to user_token_info_t.token_id'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.call_function_name IS 'API function name being called'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.related_id IS 'Related resource ID (e.g., conversation_id)'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.meta_data IS 'Additional metadata for this usage log entry, stored as JSON'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.create_time IS 'Creation time, audit field'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.update_time IS 'Update time, audit field'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.created_by IS 'Creator ID, audit field'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.updated_by IS 'Last updater ID, audit field'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.delete_flag IS 'Soft delete flag, Y means deleted'; - --- Create the ag_skill_info_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.ag_skill_info_t ( - skill_id SERIAL4 PRIMARY KEY NOT NULL, - skill_name VARCHAR(100) NOT NULL, - tenant_id VARCHAR(100), - skill_description VARCHAR(1000), - skill_tags JSON, - skill_content TEXT, - config_schemas JSON, - config_values JSON, - source VARCHAR(30) DEFAULT 'official', - created_by VARCHAR(100), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - updated_by VARCHAR(100), - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE "ag_skill_info_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.ag_skill_info_t IS 'Skill information table for managing custom skills'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_skill_info_t.skill_id IS 'Skill ID, unique primary key'; -COMMENT ON COLUMN nexent.ag_skill_info_t.skill_name IS 'Skill name, unique within tenant'; -COMMENT ON COLUMN nexent.ag_skill_info_t.tenant_id IS 'Tenant ID for multi-tenancy. NULL for pre-existing skills.'; -COMMENT ON COLUMN nexent.ag_skill_info_t.skill_description IS 'Skill description text'; -COMMENT ON COLUMN nexent.ag_skill_info_t.skill_tags IS 'Skill tags stored as JSON array'; -COMMENT ON COLUMN nexent.ag_skill_info_t.skill_content IS 'Skill content or prompt text'; -COMMENT ON COLUMN nexent.ag_skill_info_t.config_schemas IS 'Parameter metadata from config/schema.yaml'; -COMMENT ON COLUMN nexent.ag_skill_info_t.config_values IS 'Runtime parameter values from config/config.yaml'; -COMMENT ON COLUMN nexent.ag_skill_info_t.source IS 'Skill source: official, custom, or partner'; -COMMENT ON COLUMN nexent.ag_skill_info_t.created_by IS 'Creator ID'; -COMMENT ON COLUMN nexent.ag_skill_info_t.create_time IS 'Creation timestamp'; -COMMENT ON COLUMN nexent.ag_skill_info_t.updated_by IS 'Last updater ID'; -COMMENT ON COLUMN nexent.ag_skill_info_t.update_time IS 'Last update timestamp'; -COMMENT ON COLUMN nexent.ag_skill_info_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create the ag_skill_tools_rel_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.ag_skill_tools_rel_t ( - rel_id SERIAL4 PRIMARY KEY NOT NULL, - skill_id INTEGER, - tool_id INTEGER, - created_by VARCHAR(100), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - updated_by VARCHAR(100), - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE "ag_skill_tools_rel_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.ag_skill_tools_rel_t IS 'Skill-tool relationship table for many-to-many mapping'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.rel_id IS 'Relationship ID, unique primary key'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.skill_id IS 'Foreign key to ag_skill_info_t.skill_id'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.tool_id IS 'Tool ID from ag_tool_info_t'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.created_by IS 'Creator ID'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.create_time IS 'Creation timestamp'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.updated_by IS 'Last updater ID'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.update_time IS 'Last update timestamp'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create the ag_skill_instance_t table in the nexent schema --- Stores skill instance configuration per agent version --- Note: skill_description and skill_content fields removed, now retrieved from ag_skill_info_t -CREATE TABLE IF NOT EXISTS nexent.ag_skill_instance_t ( - skill_instance_id SERIAL4 NOT NULL, - skill_id INTEGER NOT NULL, - agent_id INTEGER NOT NULL, - user_id VARCHAR(100), - tenant_id VARCHAR(100), - enabled BOOLEAN DEFAULT TRUE, - version_no INTEGER DEFAULT 0 NOT NULL, - config_values JSON, - config_schemas JSON, - created_by VARCHAR(100), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - updated_by VARCHAR(100), - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N', - CONSTRAINT ag_skill_instance_t_pkey PRIMARY KEY (skill_instance_id, version_no) -); - -ALTER TABLE "ag_skill_instance_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.ag_skill_instance_t IS 'Skill instance configuration table - stores per-agent skill settings'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_skill_instance_t.skill_instance_id IS 'Skill instance ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.skill_id IS 'Foreign key to ag_skill_info_t.skill_id'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.agent_id IS 'Agent ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.enabled IS 'Whether this skill is enabled for the agent'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.config_values IS 'Per-agent runtime parameter values from config/config.yaml'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.config_schemas IS 'Per-agent parameter schema overrides from config/schema.yaml'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.created_by IS 'Creator ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.create_time IS 'Creation timestamp'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.updated_by IS 'Last updater ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.update_time IS 'Last update timestamp'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create the ag_outer_api_services table for OpenAPI services (MCP conversion) --- This table stores one record per MCP service instead of per tool -CREATE TABLE IF NOT EXISTS nexent.ag_outer_api_services ( - id BIGSERIAL PRIMARY KEY, - mcp_service_name VARCHAR(100) NOT NULL, - description TEXT, - openapi_json JSONB, - server_url VARCHAR(500), - headers_template JSONB, - tenant_id VARCHAR(100) NOT NULL, - is_available BOOLEAN DEFAULT TRUE, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.ag_outer_api_services OWNER TO "root"; - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_ag_outer_api_services_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Create a trigger to call the function before each update -CREATE TRIGGER update_ag_outer_api_services_update_time_trigger -BEFORE UPDATE ON nexent.ag_outer_api_services -FOR EACH ROW -EXECUTE FUNCTION update_ag_outer_api_services_update_time(); - --- Add comment to the table -COMMENT ON TABLE nexent.ag_outer_api_services IS 'OpenAPI services table - stores MCP service information converted from OpenAPI specs. One record per service.'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_outer_api_services.id IS 'Service ID, unique primary key'; -COMMENT ON COLUMN nexent.ag_outer_api_services.mcp_service_name IS 'MCP service name (unique identifier per tenant)'; -COMMENT ON COLUMN nexent.ag_outer_api_services.description IS 'Service description from OpenAPI info'; -COMMENT ON COLUMN nexent.ag_outer_api_services.openapi_json IS 'Complete OpenAPI JSON specification'; -COMMENT ON COLUMN nexent.ag_outer_api_services.server_url IS 'Base URL of the REST API server'; -COMMENT ON COLUMN nexent.ag_outer_api_services.headers_template IS 'Default headers template as JSONB'; -COMMENT ON COLUMN nexent.ag_outer_api_services.tenant_id IS 'Tenant ID for multi-tenancy'; -COMMENT ON COLUMN nexent.ag_outer_api_services.is_available IS 'Whether the service is available'; -COMMENT ON COLUMN nexent.ag_outer_api_services.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.ag_outer_api_services.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.ag_outer_api_services.created_by IS 'Creator'; -COMMENT ON COLUMN nexent.ag_outer_api_services.updated_by IS 'Updater'; -COMMENT ON COLUMN nexent.ag_outer_api_services.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create index for tenant_id queries -CREATE INDEX IF NOT EXISTS idx_ag_outer_api_services_tenant_id -ON nexent.ag_outer_api_services (tenant_id) -WHERE delete_flag = 'N'; - --- Create index for mcp_service_name queries -CREATE INDEX IF NOT EXISTS idx_ag_outer_api_services_mcp_service_name -ON nexent.ag_outer_api_services (mcp_service_name) -WHERE delete_flag = 'N'; - -CREATE TABLE IF NOT EXISTS nexent.ag_a2a_nacos_config_t ( - id BIGSERIAL PRIMARY KEY, - config_id VARCHAR(64) UNIQUE NOT NULL, - - nacos_addr VARCHAR(512) NOT NULL, - nacos_username VARCHAR(100), - nacos_password VARCHAR(256), - - namespace_id VARCHAR(100) DEFAULT 'public', - - name VARCHAR(100) NOT NULL, - description TEXT, - - tenant_id VARCHAR(100) NOT NULL, - created_by VARCHAR(100) NOT NULL, - updated_by VARCHAR(100), - - is_active BOOLEAN DEFAULT TRUE, - last_scan_at TIMESTAMP(6), - - create_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.ag_a2a_nacos_config_t OWNER TO "root"; - -COMMENT ON TABLE nexent.ag_a2a_nacos_config_t IS 'Nacos configuration for external A2A agent discovery. Stores connection info and discovery scope.'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.id IS 'Primary key, auto-increment'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.config_id IS 'Unique config identifier for API reference'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.nacos_addr IS 'Nacos server address, e.g., http://nacos-server:8848'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.nacos_username IS 'Nacos username for authentication'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.nacos_password IS 'Nacos password, encrypted at rest'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.namespace_id IS 'Nacos namespace for service discovery, default is public'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.name IS 'Display name for this Nacos config, e.g., Production Nacos'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.description IS 'Description of this Nacos configuration'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.tenant_id IS 'Tenant ID for multi-tenancy isolation'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.created_by IS 'User who created this config'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.updated_by IS 'User who last updated this record'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.is_active IS 'Whether this Nacos config is active'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.last_scan_at IS 'Last time a scan was performed using this config'; -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.create_time IS 'Record creation timestamp'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.update_time IS 'Record last update timestamp'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_nacos_config_t.delete_flag IS 'Soft delete flag: Y/N'; -- NOSONAR - - -CREATE TABLE IF NOT EXISTS nexent.ag_a2a_external_agent_t ( - id BIGSERIAL PRIMARY KEY, - - name VARCHAR(255) NOT NULL, - description TEXT, - version VARCHAR(50), - - agent_url VARCHAR(512) NOT NULL, - - protocol_type VARCHAR(20) DEFAULT 'JSONRPC', - - streaming BOOLEAN DEFAULT FALSE, - - supported_interfaces JSONB, - - -- Source information - source_type VARCHAR(20) NOT NULL, - - -- For URL mode: - source_url VARCHAR(512), - - -- For Nacos mode: - nacos_config_id VARCHAR(64), - nacos_agent_name VARCHAR(255), - - -- Base URL for infrastructure health checks - base_url VARCHAR(512), - - -- Tenant isolation - tenant_id VARCHAR(100) NOT NULL, - created_by VARCHAR(100) NOT NULL, - updated_by VARCHAR(100), - - raw_card JSONB, - - cached_at TIMESTAMP(6), - cache_expires_at TIMESTAMP(6), - - is_available BOOLEAN DEFAULT TRUE, - last_check_at TIMESTAMP(6), - last_check_result VARCHAR(50), - - create_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.ag_a2a_external_agent_t OWNER TO "root"; - -COMMENT ON TABLE nexent.ag_a2a_external_agent_t IS 'External A2A agents discovered from URL or Nacos. Caches Agent Cards for A2A Client role.'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.id IS 'Primary key, auto-increment. Used as unique identifier for internal references.'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.name IS 'Agent name from Agent Card'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.description IS 'Agent description from Agent Card'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.version IS 'Agent version from Agent Card, e.g., 1.2.0'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.agent_url IS 'Primary A2A endpoint URL (http-json-rpc by default, extracted from supportedInterfaces)'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.protocol_type IS 'Protocol type for calling this agent: JSONRPC, HTTP+JSON, or GRPC'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.streaming IS 'Whether this agent supports SSE streaming (from capabilities.streaming)'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.supported_interfaces IS 'All supported interfaces array from Agent Card. Format: [{protocolBinding, url, protocolVersion}, ...]'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.source_type IS 'Discovery source: url or nacos'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.source_url IS 'Direct URL to agent card (for url source type)'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.nacos_config_id IS 'Reference to Nacos config used for discovery (for nacos source type)'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.nacos_agent_name IS 'Original name used for Nacos query'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.tenant_id IS 'Tenant ID for multi-tenancy isolation'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.created_by IS 'User who discovered this agent'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.updated_by IS 'User who last updated this record'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.raw_card IS 'Full original Agent Card JSON from discovery'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.cached_at IS 'Timestamp when Agent Card was cached'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.cache_expires_at IS 'Timestamp when cache expires'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.is_available IS 'Whether this agent is currently reachable'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.last_check_at IS 'Last health check timestamp'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.last_check_result IS 'Last health check result: OK, ERROR, TIMEOUT'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.create_time IS 'Record creation timestamp'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.update_time IS 'Record last update timestamp'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.delete_flag IS 'Soft delete flag: Y/N'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.base_url IS 'Base URL for health checks (service root address)'; - - -CREATE TABLE IF NOT EXISTS nexent.ag_a2a_external_agent_relation_t ( - id BIGSERIAL PRIMARY KEY, - local_agent_id INTEGER NOT NULL, - external_agent_id BIGINT NOT NULL, - tenant_id VARCHAR(100) NOT NULL, - is_enabled BOOLEAN DEFAULT TRUE, - created_by VARCHAR(100) NOT NULL, - updated_by VARCHAR(100), - create_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N', - CONSTRAINT uq_local_external_agent UNIQUE (local_agent_id, external_agent_id) -); - -ALTER TABLE nexent.ag_a2a_external_agent_relation_t OWNER TO "root"; - -COMMENT ON TABLE nexent.ag_a2a_external_agent_relation_t IS 'Relation between local agent and external A2A agent. Enables local agents to call external A2A agents as sub-agents.'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_relation_t.id IS 'Primary key, auto-increment'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_external_agent_relation_t.local_agent_id IS 'Local parent agent ID (FK to ag_tenant_agent_t)'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_relation_t.external_agent_id IS 'External A2A agent ID (FK to ag_a2a_external_agent_t.id)'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_relation_t.tenant_id IS 'Tenant ID for multi-tenancy isolation'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_external_agent_relation_t.is_enabled IS 'Whether this relation is active'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_relation_t.created_by IS 'User who created this relation'; -COMMENT ON COLUMN nexent.ag_a2a_external_agent_relation_t.updated_by IS 'User who last updated this record'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_external_agent_relation_t.create_time IS 'Record creation timestamp'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_external_agent_relation_t.update_time IS 'Record last update timestamp'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_external_agent_relation_t.delete_flag IS 'Soft delete flag: Y/N'; -- NOSONAR - -CREATE TABLE IF NOT EXISTS nexent.ag_a2a_server_agent_t ( - id BIGSERIAL PRIMARY KEY, - agent_id INTEGER NOT NULL, - user_id VARCHAR(100) NOT NULL, - tenant_id VARCHAR(100) NOT NULL, - created_by VARCHAR(100), - updated_by VARCHAR(100), - endpoint_id VARCHAR(64) UNIQUE NOT NULL, - name VARCHAR(255) NOT NULL, - description TEXT, - version VARCHAR(50), - agent_url VARCHAR(512), - streaming BOOLEAN DEFAULT FALSE, - supported_interfaces JSONB, - card_overrides JSONB, - is_enabled BOOLEAN DEFAULT FALSE, - raw_card JSONB, - published_at TIMESTAMP(6), - unpublished_at TIMESTAMP(6), - response_format VARCHAR(20) DEFAULT 'task', - create_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.ag_a2a_server_agent_t OWNER TO "root"; - -COMMENT ON TABLE nexent.ag_a2a_server_agent_t IS 'Local agents registered as A2A Server endpoints. Exposes Agent Cards for external A2A callers.'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.id IS 'Primary key, auto-increment'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.agent_id IS 'Local agent ID (FK to ag_tenant_agent_t)'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.user_id IS 'Owner user ID'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.tenant_id IS 'Tenant ID for multi-tenancy isolation'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.created_by IS 'User who created this A2A Server agent'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.updated_by IS 'User who last updated this A2A Server agent'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.endpoint_id IS 'Generated endpoint ID, format: a2a_{agent_id[:8]}_{hash[:8]}'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.name IS 'Agent name exposed in Agent Card (from agent or override)'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.description IS 'Agent description exposed in Agent Card'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.version IS 'Agent version exposed in Agent Card'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.agent_url IS 'Primary A2A endpoint URL (http-json-rpc by default)'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.streaming IS 'Whether this agent supports SSE streaming'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.supported_interfaces IS 'All supported interfaces: [{protocolBinding, url, protocolVersion}, ...]'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.card_overrides IS 'User customizations for Agent Card (partial override)'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.is_enabled IS 'Whether A2A Server is enabled for this agent'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.raw_card IS 'Generated Agent Card JSON (for debugging)'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.published_at IS 'Timestamp when A2A Server was last enabled'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.unpublished_at IS 'Timestamp when A2A Server was disabled'; -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.create_time IS 'Record creation timestamp'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.update_time IS 'Record last update timestamp'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.delete_flag IS 'Soft delete flag: Y/N'; -- NOSONAR -COMMENT ON COLUMN nexent.ag_a2a_server_agent_t.response_format IS 'Response format: ''task'' for full Task response, ''message'' for simple Message response'; - - -CREATE TABLE IF NOT EXISTS nexent.ag_a2a_task_t ( - id VARCHAR(64) PRIMARY KEY, -- taskId - context_id VARCHAR(64), -- contextId - endpoint_id VARCHAR(64) NOT NULL, - caller_user_id VARCHAR(100), - caller_tenant_id VARCHAR(100), - raw_request JSONB, - task_state VARCHAR(50) NOT NULL DEFAULT 'TASK_STATE_SUBMITTED', - state_timestamp TIMESTAMP(6), -- State update timestamp - result_data JSONB, -- Final result (renamed from result to avoid SQL function conflict) - create_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - completed_at TIMESTAMP(6) -); - -ALTER TABLE nexent.ag_a2a_task_t OWNER TO "root"; - -COMMENT ON TABLE nexent.ag_a2a_task_t IS 'A2A tasks for tracking requests. Task is the unit of work, not all requests need to create a task.'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.id IS 'Task ID from A2A protocol, primary key'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.context_id IS 'Context ID for grouping related A2A tasks'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.endpoint_id IS 'Endpoint ID (FK to ag_a2a_server_agent_t.endpoint_id)'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.caller_user_id IS 'User ID of the caller (for audit)'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.caller_tenant_id IS 'Tenant ID of the caller (for audit)'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.raw_request IS 'Original A2A request payload'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.task_state IS 'Task state: TASK_STATE_SUBMITTED, TASK_STATE_WORKING, TASK_STATE_COMPLETED, TASK_STATE_FAILED, TASK_STATE_CANCELED, TASK_STATE_INPUT_REQUIRED, TASK_STATE_REJECTED, TASK_STATE_AUTH_REQUIRED'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.state_timestamp IS 'Task state last update timestamp'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.result_data IS 'Task final result data'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.create_time IS 'Task creation timestamp'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.update_time IS 'Task last update timestamp'; -COMMENT ON COLUMN nexent.ag_a2a_task_t.completed_at IS 'Task completion timestamp'; - -CREATE TABLE IF NOT EXISTS nexent.ag_a2a_message_t ( - message_id VARCHAR(64) PRIMARY KEY, -- messageId (A2A spec naming) - task_id VARCHAR(64), -- taskId (associated task), can be NULL for simple requests - message_index INTEGER NOT NULL, -- Sequence index - role VARCHAR(20) NOT NULL CHECK (role IN ('ROLE_UNSPECIFIED', 'ROLE_USER', 'ROLE_AGENT')), -- Following A2A spec: ROLE_UNSPECIFIED, ROLE_USER, ROLE_AGENT - parts JSONB NOT NULL, -- Part array - meta_data JSONB, -- Optional metadata - extensions JSONB, -- Extension URI list - reference_task_ids JSONB, -- Referenced task IDs array - create_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - UNIQUE(task_id, message_index) -); - -ALTER TABLE nexent.ag_a2a_message_t OWNER TO "root"; - -COMMENT ON TABLE nexent.ag_a2a_message_t IS 'A2A messages within tasks. Stores conversation history for multi-turn interactions.'; -COMMENT ON COLUMN nexent.ag_a2a_message_t.message_id IS 'Message ID, primary key (A2A spec: messageId)'; -COMMENT ON COLUMN nexent.ag_a2a_message_t.task_id IS 'Task ID this message belongs to (FK to ag_a2a_task_t.id), can be NULL for simple requests without Task'; -COMMENT ON COLUMN nexent.ag_a2a_message_t.message_index IS 'Order of message in the conversation'; -COMMENT ON COLUMN nexent.ag_a2a_message_t.role IS 'Message sender role: ROLE_UNSPECIFIED, ROLE_USER, or ROLE_AGENT'; -COMMENT ON COLUMN nexent.ag_a2a_message_t.parts IS 'Message parts following A2A Part structure: [{"type": "text", "text": "..."}]'; -COMMENT ON COLUMN nexent.ag_a2a_message_t.meta_data IS 'Optional message metadata'; -COMMENT ON COLUMN nexent.ag_a2a_message_t.extensions IS 'Extension URI list'; -COMMENT ON COLUMN nexent.ag_a2a_message_t.reference_task_ids IS 'Referenced task IDs array for multi-turn scenarios'; -COMMENT ON COLUMN nexent.ag_a2a_message_t.create_time IS 'Message creation timestamp'; - -CREATE TABLE IF NOT EXISTS nexent.ag_a2a_artifact_t ( - id VARCHAR(64) PRIMARY KEY, -- Internal primary key - artifact_id VARCHAR(64) NOT NULL, -- artifactId (A2A spec naming) - task_id VARCHAR(64) NOT NULL, -- taskId (associated task, required) - name VARCHAR(255), -- Human-readable name - description TEXT, -- Description - parts JSONB NOT NULL, -- Part array (following A2A spec) - meta_data JSONB, -- Metadata - extensions JSONB, -- Extension URI list - create_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - UNIQUE(task_id, artifact_id) -); - -ALTER TABLE nexent.ag_a2a_artifact_t OWNER TO "root"; - -COMMENT ON TABLE nexent.ag_a2a_artifact_t IS 'A2A artifacts. Stores the output/artifacts produced by a task.'; -COMMENT ON COLUMN nexent.ag_a2a_artifact_t.id IS 'Internal primary key'; -COMMENT ON COLUMN nexent.ag_a2a_artifact_t.artifact_id IS 'Artifact ID (A2A spec: artifactId)'; -COMMENT ON COLUMN nexent.ag_a2a_artifact_t.task_id IS 'Task ID this artifact belongs to (FK to ag_a2a_task_t.id), required - no standalone artifacts'; -COMMENT ON COLUMN nexent.ag_a2a_artifact_t.name IS 'Human-readable artifact name'; -COMMENT ON COLUMN nexent.ag_a2a_artifact_t.description IS 'Artifact description'; -COMMENT ON COLUMN nexent.ag_a2a_artifact_t.parts IS 'Artifact parts following A2A Part structure: [{"type": "text", "text": "..."}]'; -COMMENT ON COLUMN nexent.ag_a2a_artifact_t.meta_data IS 'Artifact metadata'; -COMMENT ON COLUMN nexent.ag_a2a_artifact_t.extensions IS 'Extension URI list'; -COMMENT ON COLUMN nexent.ag_a2a_artifact_t.create_time IS 'Artifact creation timestamp'; - --- Create the model_monitoring_record_t table for LLM performance metrics -CREATE TABLE IF NOT EXISTS nexent.model_monitoring_record_t ( - monitoring_id SERIAL PRIMARY KEY, - model_id INT4, - model_name VARCHAR(100) NOT NULL, - model_type VARCHAR(20) DEFAULT 'llm', - agent_id INT4, - agent_name VARCHAR(100), - conversation_id INT4, - tenant_id VARCHAR(100) NOT NULL, - user_id VARCHAR(100), - display_name VARCHAR(100), - request_duration_ms INT4, - ttft_ms INT4, - input_tokens INT4, - output_tokens INT4, - total_tokens INT4, - generation_rate FLOAT, - is_streaming BOOLEAN DEFAULT FALSE, - is_success BOOLEAN DEFAULT TRUE, - is_error BOOLEAN DEFAULT FALSE, - error_type VARCHAR(50), - error_message TEXT, - retry_count INT4 DEFAULT 0, - operation VARCHAR(50), - create_time TIMESTAMP DEFAULT NOW(), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.model_monitoring_record_t OWNER TO "root"; - -COMMENT ON TABLE nexent.model_monitoring_record_t IS 'Per-request LLM performance metrics for model monitoring'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.monitoring_id IS 'Monitoring record ID, unique primary key'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.model_id IS 'Foreign key to model_record_t.model_id'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.model_name IS 'Model identifier (repo/name format)'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.model_type IS 'Model type: llm, vlm, embedding, multi_embedding, rerank'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.agent_id IS 'Agent ID that initiated the request'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.agent_name IS 'Agent display name'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.conversation_id IS 'Conversation ID associated with the request'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.tenant_id IS 'Tenant ID for multi-tenancy isolation'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.user_id IS 'User ID who initiated the request'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.display_name IS 'Human-readable model display name'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.request_duration_ms IS 'Total request duration in milliseconds'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.ttft_ms IS 'Time to first token in milliseconds (streaming only)'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.input_tokens IS 'Number of input prompt tokens'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.output_tokens IS 'Number of output completion tokens'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.total_tokens IS 'Total tokens (input + output)'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.generation_rate IS 'Token generation rate in tokens per second'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.is_streaming IS 'Whether the request used streaming response'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.is_success IS 'Whether the request completed successfully'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.is_error IS 'Whether the request resulted in an error'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.error_type IS 'Error exception class name'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.error_message IS 'Error message text'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.retry_count IS 'Number of retry attempts'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.operation IS 'Operation type: chat_completion, title_generation, connectivity_check, embedding_call, system_prompt_generation'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.create_time IS 'Record creation timestamp'; -COMMENT ON COLUMN nexent.model_monitoring_record_t.delete_flag IS 'Soft delete flag: Y/N'; - -CREATE INDEX IF NOT EXISTS ix_monitoring_model_id ON nexent.model_monitoring_record_t (model_id); -CREATE INDEX IF NOT EXISTS ix_monitoring_tenant_id ON nexent.model_monitoring_record_t (tenant_id); -CREATE INDEX IF NOT EXISTS ix_monitoring_agent_id ON nexent.model_monitoring_record_t (agent_id); -CREATE INDEX IF NOT EXISTS ix_monitoring_create_time ON nexent.model_monitoring_record_t (create_time); -CREATE INDEX IF NOT EXISTS ix_monitoring_is_error ON nexent.model_monitoring_record_t (is_error); -CREATE INDEX IF NOT EXISTS ix_monitoring_model_type ON nexent.model_monitoring_record_t (model_type); -CREATE INDEX IF NOT EXISTS ix_monitoring_model_time ON nexent.model_monitoring_record_t (model_id, create_time); - --- Create user OAuth account table for third-party login (GitHub, WeChat, etc.) -CREATE TABLE IF NOT EXISTS nexent.user_oauth_account_t ( - oauth_account_id SERIAL PRIMARY KEY, - user_id VARCHAR(100) NOT NULL, - provider VARCHAR(30) NOT NULL, - provider_user_id VARCHAR(200) NOT NULL, - provider_email VARCHAR(255), - provider_username VARCHAR(200), - tenant_id VARCHAR(100), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag CHAR(1) DEFAULT 'N', - CONSTRAINT uq_oauth_provider_user UNIQUE (provider, provider_user_id) -); - -ALTER TABLE nexent.user_oauth_account_t OWNER TO "root"; - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_user_oauth_account_t_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Create a trigger to call the function before each update -CREATE TRIGGER update_user_oauth_account_t_update_time_trigger -BEFORE UPDATE ON nexent.user_oauth_account_t -FOR EACH ROW -EXECUTE FUNCTION update_user_oauth_account_t_update_time(); - --- Add comments -COMMENT ON TABLE nexent.user_oauth_account_t IS 'User OAuth account table - third-party login bindings'; -COMMENT ON COLUMN nexent.user_oauth_account_t.oauth_account_id IS 'OAuth account ID, primary key'; -COMMENT ON COLUMN nexent.user_oauth_account_t.user_id IS 'Nexent user ID (Supabase UUID)'; -COMMENT ON COLUMN nexent.user_oauth_account_t.provider IS 'OAuth provider name: github, wechat, gde, link_app'; -COMMENT ON COLUMN nexent.user_oauth_account_t.provider_user_id IS 'User ID from the OAuth provider'; -COMMENT ON COLUMN nexent.user_oauth_account_t.provider_email IS 'Email from the OAuth provider'; -COMMENT ON COLUMN nexent.user_oauth_account_t.provider_username IS 'Display name from the OAuth provider'; -COMMENT ON COLUMN nexent.user_oauth_account_t.tenant_id IS 'Tenant ID at time of linking'; -COMMENT ON COLUMN nexent.user_oauth_account_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.user_oauth_account_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.user_oauth_account_t.created_by IS 'Creator'; -COMMENT ON COLUMN nexent.user_oauth_account_t.updated_by IS 'Updater'; -COMMENT ON COLUMN nexent.user_oauth_account_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create index for user_id queries -CREATE INDEX IF NOT EXISTS idx_user_oauth_account_t_user_id -ON nexent.user_oauth_account_t (user_id); - --- mcp_community_record_t: Community MCP market table -CREATE TABLE IF NOT EXISTS nexent.mcp_community_record_t ( - community_id SERIAL PRIMARY KEY NOT NULL, - tenant_id VARCHAR(100), - user_id VARCHAR(100), - mcp_name VARCHAR(100) NOT NULL, - mcp_server VARCHAR(500) NOT NULL, - source VARCHAR(30) DEFAULT 'community', - version VARCHAR(50), - registry_json JSONB, - transport_type VARCHAR(30), - config_json JSON, - tags TEXT[], - description TEXT, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.mcp_community_record_t OWNER TO root; - -COMMENT ON TABLE nexent.mcp_community_record_t IS 'Community MCP market records, publishable from tenant MCP services'; -COMMENT ON COLUMN nexent.mcp_community_record_t.community_id IS 'Community record ID, unique primary key'; -COMMENT ON COLUMN nexent.mcp_community_record_t.tenant_id IS 'Publisher tenant ID'; -COMMENT ON COLUMN nexent.mcp_community_record_t.user_id IS 'Publisher user ID'; -COMMENT ON COLUMN nexent.mcp_community_record_t.mcp_name IS 'MCP name'; -COMMENT ON COLUMN nexent.mcp_community_record_t.mcp_server IS 'MCP server URL'; -COMMENT ON COLUMN nexent.mcp_community_record_t.source IS 'Source type, fixed to community for this table'; -COMMENT ON COLUMN nexent.mcp_community_record_t.version IS 'MCP version'; -COMMENT ON COLUMN nexent.mcp_community_record_t.registry_json IS 'Full MCP server metadata JSON for discovery and quick import'; -COMMENT ON COLUMN nexent.mcp_community_record_t.transport_type IS 'Transport type: url/container'; -COMMENT ON COLUMN nexent.mcp_community_record_t.config_json IS 'Public-shareable MCP configuration JSON'; -COMMENT ON COLUMN nexent.mcp_community_record_t.tags IS 'Tags'; -COMMENT ON COLUMN nexent.mcp_community_record_t.description IS 'Description'; -COMMENT ON COLUMN nexent.mcp_community_record_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.mcp_community_record_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.mcp_community_record_t.created_by IS 'Creator ID'; -COMMENT ON COLUMN nexent.mcp_community_record_t.updated_by IS 'Updater ID'; -COMMENT ON COLUMN nexent.mcp_community_record_t.delete_flag IS 'Soft delete flag: Y/N'; - -CREATE INDEX IF NOT EXISTS idx_mcp_community_tenant_delete - ON nexent.mcp_community_record_t (tenant_id, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_community_name_delete - ON nexent.mcp_community_record_t (mcp_name, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_community_transport_delete - ON nexent.mcp_community_record_t (transport_type, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_community_user_delete - ON nexent.mcp_community_record_t (user_id, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_community_tags_gin - ON nexent.mcp_community_record_t USING GIN (tags); - -CREATE OR REPLACE FUNCTION update_mcp_community_record_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -COMMENT ON FUNCTION update_mcp_community_record_update_time() IS 'Auto-update update_time for mcp_community_record_t'; - -DROP TRIGGER IF EXISTS update_mcp_community_record_update_time_trigger ON nexent.mcp_community_record_t; -CREATE TRIGGER update_mcp_community_record_update_time_trigger -BEFORE UPDATE ON nexent.mcp_community_record_t -FOR EACH ROW -EXECUTE FUNCTION update_mcp_community_record_update_time(); - -COMMENT ON TRIGGER update_mcp_community_record_update_time_trigger ON nexent.mcp_community_record_t IS 'Trigger to maintain update_time'; - -CREATE TABLE IF NOT EXISTS nexent.user_cas_session_t ( - cas_session_id SERIAL PRIMARY KEY, - session_id VARCHAR(100) NOT NULL UNIQUE, - user_id VARCHAR(100) NOT NULL, - cas_user_id VARCHAR(200) NOT NULL, - cas_session_index VARCHAR(500), - status VARCHAR(30) NOT NULL DEFAULT 'active', - expires_at TIMESTAMP NOT NULL, - revoked_at TIMESTAMP, - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -CREATE INDEX IF NOT EXISTS ix_user_cas_session_session_id - ON nexent.user_cas_session_t (session_id); -CREATE INDEX IF NOT EXISTS ix_user_cas_session_user_id - ON nexent.user_cas_session_t (user_id); -CREATE INDEX IF NOT EXISTS ix_user_cas_session_cas_user_id - ON nexent.user_cas_session_t (cas_user_id); - -COMMENT ON TABLE nexent.user_cas_session_t IS 'Server-side session records for CAS SSO login and logout synchronization'; -COMMENT ON COLUMN nexent.user_cas_session_t.session_id IS 'JWT sid claim for revocation checks'; -COMMENT ON COLUMN nexent.user_cas_session_t.cas_user_id IS 'User identifier returned by CAS'; -COMMENT ON COLUMN nexent.user_cas_session_t.cas_session_index IS 'CAS SessionIndex or service ticket'; diff --git a/docker/sql/v1.1.0_0619_add_tenant_config_t.sql b/docker/sql/v1.1.0_0619_add_tenant_config_t.sql deleted file mode 100644 index b2079101c..000000000 --- a/docker/sql/v1.1.0_0619_add_tenant_config_t.sql +++ /dev/null @@ -1,65 +0,0 @@ --- 1. 为knowledge_record_t表添加knowledge_sources列 -ALTER TABLE nexent.knowledge_record_t -ADD COLUMN IF NOT EXISTS "knowledge_sources" varchar(100) COLLATE "pg_catalog"."default"; - --- 添加列注释 -COMMENT ON COLUMN nexent.knowledge_record_t."knowledge_sources" IS 'Knowledge base sources'; - - --- 2. 创建tenant_config_t表 -CREATE TABLE IF NOT EXISTS nexent.tenant_config_t ( - tenant_config_id SERIAL PRIMARY KEY NOT NULL, - tenant_id VARCHAR(100), - user_id VARCHAR(100), - value_type VARCHAR(100), - config_key VARCHAR(100), - config_value VARCHAR(10000), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- 添加表注释 -COMMENT ON TABLE nexent.tenant_config_t IS 'Tenant configuration information table'; - --- 添加列注释 -COMMENT ON COLUMN nexent.tenant_config_t.tenant_config_id IS 'ID'; -COMMENT ON COLUMN nexent.tenant_config_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.tenant_config_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.tenant_config_t.value_type IS 'Value type'; -COMMENT ON COLUMN nexent.tenant_config_t.config_key IS 'Config key'; -COMMENT ON COLUMN nexent.tenant_config_t.config_value IS 'Config value'; -COMMENT ON COLUMN nexent.tenant_config_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.tenant_config_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.tenant_config_t.created_by IS 'Creator'; -COMMENT ON COLUMN nexent.tenant_config_t.updated_by IS 'Updater'; -COMMENT ON COLUMN nexent.tenant_config_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- 创建更新update_time的函数 -CREATE OR REPLACE FUNCTION update_tenant_config_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- 添加函数注释 -COMMENT ON FUNCTION update_tenant_config_update_time() IS 'Function to update the update_time column when a record in tenant_config_t is updated'; - --- 创建触发器 -DROP TRIGGER IF EXISTS update_tenant_config_update_time_trigger ON nexent.tenant_config_t; -CREATE TRIGGER update_tenant_config_update_time_trigger -BEFORE UPDATE ON nexent.tenant_config_t -FOR EACH ROW -EXECUTE FUNCTION update_tenant_config_update_time(); - --- 添加触发器注释 -COMMENT ON TRIGGER update_tenant_config_update_time_trigger ON nexent.tenant_config_t -IS 'Trigger to call update_tenant_config_update_time function before each update on tenant_config_t table'; - -ALTER TABLE model_record_t -ADD COLUMN IF NOT EXISTS tenant_id varchar(100) COLLATE pg_catalog.default DEFAULT 'tenant_id'; -COMMENT ON COLUMN "model_record_t"."tenant_id" IS 'Tenant ID for filtering'; \ No newline at end of file diff --git a/docker/sql/v1.2.0_0627_increase_config_value_length.sql b/docker/sql/v1.2.0_0627_increase_config_value_length.sql deleted file mode 100644 index ae427c0a8..000000000 --- a/docker/sql/v1.2.0_0627_increase_config_value_length.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Incremental SQL to alter config_value column length in nexent.tenant_config_t table - --- Check if the table exists before attempting to alter it -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = 'nexent' - AND table_name = 'tenant_config_t' - ) THEN - -- Alter the column length - EXECUTE 'ALTER TABLE nexent.tenant_config_t ALTER COLUMN config_value TYPE VARCHAR(10000)'; - - -- Log the change - RAISE NOTICE 'Altered config_value column length from VARCHAR(100) to VARCHAR(10000) in nexent.tenant_config_t'; - ELSE - RAISE NOTICE 'Table nexent.tenant_config_t does not exist, skipping alteration'; - END IF; -END $$; \ No newline at end of file diff --git a/docker/sql/v1.3.0_0630_add_mcp_record_t.sql b/docker/sql/v1.3.0_0630_add_mcp_record_t.sql deleted file mode 100644 index 3f25a5957..000000000 --- a/docker/sql/v1.3.0_0630_add_mcp_record_t.sql +++ /dev/null @@ -1,59 +0,0 @@ --- Migration: Add mcp_record_t table --- Date: 2024-06-30 --- Description: Create MCP (Model Context Protocol) records table with audit fields - --- Set search path to nexent schema -SET search_path TO nexent; - --- Create the mcp_record_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.mcp_record_t ( - mcp_id SERIAL PRIMARY KEY NOT NULL, - tenant_id VARCHAR(100), - user_id VARCHAR(100), - mcp_name VARCHAR(100), - mcp_server VARCHAR(500), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE "mcp_record_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.mcp_record_t IS 'MCP (Model Context Protocol) records table'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.mcp_record_t.mcp_id IS 'MCP record ID, unique primary key'; -COMMENT ON COLUMN nexent.mcp_record_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.mcp_record_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.mcp_record_t.mcp_name IS 'MCP name'; -COMMENT ON COLUMN nexent.mcp_record_t.mcp_server IS 'MCP server address'; -COMMENT ON COLUMN nexent.mcp_record_t.create_time IS 'Creation time, audit field'; -COMMENT ON COLUMN nexent.mcp_record_t.update_time IS 'Update time, audit field'; -COMMENT ON COLUMN nexent.mcp_record_t.created_by IS 'Creator ID, audit field'; -COMMENT ON COLUMN nexent.mcp_record_t.updated_by IS 'Last updater ID, audit field'; -COMMENT ON COLUMN nexent.mcp_record_t.delete_flag IS 'When deleted by user frontend, delete flag will be set to true, achieving soft delete effect. Optional values Y/N'; - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_mcp_record_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Add comment to the function -COMMENT ON FUNCTION update_mcp_record_update_time() IS 'Function to update the update_time column when a record in mcp_record_t is updated'; - --- Create a trigger to call the function before each update -DROP TRIGGER IF EXISTS update_mcp_record_update_time_trigger ON nexent.mcp_record_t; -CREATE TRIGGER update_mcp_record_update_time_trigger -BEFORE UPDATE ON nexent.mcp_record_t -FOR EACH ROW -EXECUTE FUNCTION update_mcp_record_update_time(); - --- Add comment to the trigger -COMMENT ON TRIGGER update_mcp_record_update_time_trigger ON nexent.mcp_record_t IS 'Trigger to call update_mcp_record_update_time function before each update on mcp_record_t table'; diff --git a/docker/sql/v1.4.0_0708_add_user_tenant_t.sql b/docker/sql/v1.4.0_0708_add_user_tenant_t.sql deleted file mode 100644 index 253c8b370..000000000 --- a/docker/sql/v1.4.0_0708_add_user_tenant_t.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Create user tenant relationship table -CREATE TABLE IF NOT EXISTS nexent.user_tenant_t ( - user_tenant_id SERIAL PRIMARY KEY, - user_id VARCHAR(100) NOT NULL, - tenant_id VARCHAR(100) NOT NULL, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag CHAR(1) DEFAULT 'N', - UNIQUE(user_id, tenant_id) -); - --- Add comment -COMMENT ON TABLE nexent.user_tenant_t IS 'User tenant relationship table'; -COMMENT ON COLUMN nexent.user_tenant_t.user_tenant_id IS 'User tenant relationship ID, primary key'; -COMMENT ON COLUMN nexent.user_tenant_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.user_tenant_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.user_tenant_t.create_time IS 'Create time'; -COMMENT ON COLUMN nexent.user_tenant_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.user_tenant_t.created_by IS 'Created by'; -COMMENT ON COLUMN nexent.user_tenant_t.updated_by IS 'Updated by'; -COMMENT ON COLUMN nexent.user_tenant_t.delete_flag IS 'Delete flag, Y/N'; \ No newline at end of file diff --git a/docker/sql/v1.5.0_0715_add_knowledge_describe_length.sql b/docker/sql/v1.5.0_0715_add_knowledge_describe_length.sql deleted file mode 100644 index 95988150e..000000000 --- a/docker/sql/v1.5.0_0715_add_knowledge_describe_length.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE nexent.knowledge_record_t - ALTER COLUMN knowledge_describe TYPE varchar(3000); \ No newline at end of file diff --git a/docker/sql/v1.5.0_0716_add_status_to_mcp_record_t.sql b/docker/sql/v1.5.0_0716_add_status_to_mcp_record_t.sql deleted file mode 100644 index ac233a8bf..000000000 --- a/docker/sql/v1.5.0_0716_add_status_to_mcp_record_t.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE nexent.mcp_record_t -ADD COLUMN IF NOT EXISTS status BOOLEAN DEFAULT NULL; -COMMENT ON COLUMN nexent.mcp_record_t.status IS 'MCP server connection status, true=connected, false=disconnected, null=unknown'; \ No newline at end of file diff --git a/docker/sql/v1.6.0_0722_modify_tenant_agent.sql b/docker/sql/v1.6.0_0722_modify_tenant_agent.sql deleted file mode 100644 index cce2c433e..000000000 --- a/docker/sql/v1.6.0_0722_modify_tenant_agent.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Migration script to add new prompt fields to ag_tenant_agent_t table --- Add three new columns for storing segmented prompt content - --- Add duty_prompt column -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS duty_prompt TEXT; - --- Add constraint_prompt column -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS constraint_prompt TEXT; - --- Add few_shots_prompt column -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS few_shots_prompt TEXT; - --- Drop prompt column -ALTER TABLE nexent.ag_tenant_agent_t -DROP COLUMN IF EXISTS prompt; - --- Add comments to the new columns -COMMENT ON COLUMN nexent.ag_tenant_agent_t.duty_prompt IS 'Duty prompt content'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.constraint_prompt IS 'Constraint prompt content'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.few_shots_prompt IS 'Few shots prompt content'; \ No newline at end of file diff --git a/docker/sql/v1.6.0_0723_add_agent_relation_t.sql b/docker/sql/v1.6.0_0723_add_agent_relation_t.sql deleted file mode 100644 index 78d856438..000000000 --- a/docker/sql/v1.6.0_0723_add_agent_relation_t.sql +++ /dev/null @@ -1,45 +0,0 @@ --- Migration script to add ag_agent_relation_t table for recording agent parent-child relationships --- This table is used to store the hierarchical relationships between agents - --- Create the ag_agent_relation_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.ag_agent_relation_t ( - relation_id SERIAL PRIMARY KEY NOT NULL, - selected_agent_id INTEGER, - parent_agent_id INTEGER, - tenant_id VARCHAR(100), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_ag_agent_relation_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Create a trigger to call the function before each update -DROP TRIGGER IF EXISTS update_ag_agent_relation_update_time_trigger ON nexent.ag_agent_relation_t; -CREATE TRIGGER update_ag_agent_relation_update_time_trigger -BEFORE UPDATE ON nexent.ag_agent_relation_t -FOR EACH ROW -EXECUTE FUNCTION update_ag_agent_relation_update_time(); - --- Add comment to the table -COMMENT ON TABLE nexent.ag_agent_relation_t IS 'Agent parent-child relationship table'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_agent_relation_t.relation_id IS 'Relationship ID, primary key'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.selected_agent_id IS 'Selected agent ID'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.parent_agent_id IS 'Parent agent ID'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.create_time IS 'Creation time, audit field'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.update_time IS 'Update time, audit field'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.created_by IS 'Creator ID, audit field'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.updated_by IS 'Last updater ID, audit field'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.delete_flag IS 'Delete flag, set to Y for soft delete, optional values Y/N'; \ No newline at end of file diff --git a/docker/sql/v1.7.1_0805_add_deep_thinking_to_model_record_t.sql b/docker/sql/v1.7.1_0805_add_deep_thinking_to_model_record_t.sql deleted file mode 100644 index 65b5b8465..000000000 --- a/docker/sql/v1.7.1_0805_add_deep_thinking_to_model_record_t.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE nexent.model_record_t -ADD COLUMN IF NOT EXISTS is_deep_thinking BOOLEAN DEFAULT FALSE; -COMMENT ON COLUMN nexent.model_record_t.is_deep_thinking IS 'deep thinking switch, true=open, false=close'; \ No newline at end of file diff --git a/docker/sql/v1.7.1_0806_add_memory_user_config.sql b/docker/sql/v1.7.1_0806_add_memory_user_config.sql deleted file mode 100644 index 46eb42829..000000000 --- a/docker/sql/v1.7.1_0806_add_memory_user_config.sql +++ /dev/null @@ -1,54 +0,0 @@ --- 创建序列 -CREATE SEQUENCE IF NOT EXISTS "nexent"."memory_user_config_t_config_id_seq" -INCREMENT 1 -MINVALUE 1 -MAXVALUE 2147483647 -START 1 -CACHE 1; - - --- 创建表 -CREATE TABLE IF NOT EXISTS "nexent"."memory_user_config_t" ( - "config_id" SERIAL PRIMARY KEY NOT NULL, - "tenant_id" varchar(100) COLLATE "pg_catalog"."default", - "user_id" varchar(100) COLLATE "pg_catalog"."default", - "value_type" varchar(100) COLLATE "pg_catalog"."default", - "config_key" varchar(100) COLLATE "pg_catalog"."default", - "config_value" varchar(100) COLLATE "pg_catalog"."default", - "create_time" timestamp(6) DEFAULT CURRENT_TIMESTAMP, - "update_time" timestamp(6) DEFAULT CURRENT_TIMESTAMP, - "created_by" varchar(100) COLLATE "pg_catalog"."default", - "updated_by" varchar(100) COLLATE "pg_catalog"."default", - "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying -); - --- 设置表所有者 -ALTER TABLE "nexent"."memory_user_config_t" OWNER TO "root"; - -COMMENT ON COLUMN "nexent"."memory_user_config_t"."config_id" IS 'ID'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."tenant_id" IS 'Tenant ID'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."user_id" IS 'User ID'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."value_type" IS 'Value type. Optional values: single/multi'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."config_key" IS 'Config key'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."config_value" IS 'Config value'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."create_time" IS 'Creation time'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."update_time" IS 'Update time'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."created_by" IS 'Creator'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."updated_by" IS 'Updater'; -COMMENT ON COLUMN "nexent"."memory_user_config_t"."delete_flag" IS 'Whether it is deleted. Optional values: Y/N'; - -COMMENT ON TABLE "nexent"."memory_user_config_t" IS 'User configuration of memory setting table'; - -CREATE OR REPLACE FUNCTION "update_memory_user_config_update_time"() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS "update_memory_user_config_update_time_trigger" ON "nexent"."memory_user_config_t"; -CREATE TRIGGER "update_memory_user_config_update_time_trigger" -BEFORE UPDATE ON "nexent"."memory_user_config_t" -FOR EACH ROW -EXECUTE FUNCTION "update_memory_user_config_update_time"(); \ No newline at end of file diff --git a/docker/sql/v1.7.2.2_0820_add_partner_mapping_id_t.sql b/docker/sql/v1.7.2.2_0820_add_partner_mapping_id_t.sql deleted file mode 100644 index 4817b6afc..000000000 --- a/docker/sql/v1.7.2.2_0820_add_partner_mapping_id_t.sql +++ /dev/null @@ -1,48 +0,0 @@ -CREATE SEQUENCE IF NOT EXISTS "nexent"."partner_mapping_id_t_mapping_id_seq" -INCREMENT 1 -MINVALUE 1 -MAXVALUE 2147483647 -START 1 -CACHE 1; - -CREATE TABLE IF NOT EXISTS "nexent"."partner_mapping_id_t" ( - "mapping_id" serial PRIMARY KEY NOT NULL, - "external_id" varchar(100) COLLATE "pg_catalog"."default", - "internal_id" int4, - "mapping_type" varchar(30) COLLATE "pg_catalog"."default", - "tenant_id" varchar(100) COLLATE "pg_catalog"."default", - "user_id" varchar(100) COLLATE "pg_catalog"."default", - "create_time" timestamp(6) DEFAULT CURRENT_TIMESTAMP, - "update_time" timestamp(6) DEFAULT CURRENT_TIMESTAMP, - "created_by" varchar(100) COLLATE "pg_catalog"."default", - "updated_by" varchar(100) COLLATE "pg_catalog"."default", - "delete_flag" varchar(1) COLLATE "pg_catalog"."default" DEFAULT 'N'::character varying -); - -ALTER TABLE "nexent"."partner_mapping_id_t" OWNER TO "root"; - -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."mapping_id" IS 'ID'; -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."external_id" IS 'The external id given by the outer partner'; -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."internal_id" IS 'The internal id of the other database table'; -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."mapping_type" IS 'Type of the external - internal mapping, value set: CONVERSATION'; -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."tenant_id" IS 'Tenant ID'; -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."user_id" IS 'User ID'; -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."create_time" IS 'Creation time'; -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."update_time" IS 'Update time'; -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."created_by" IS 'Creator'; -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."updated_by" IS 'Updater'; -COMMENT ON COLUMN "nexent"."partner_mapping_id_t"."delete_flag" IS 'Whether it is deleted. Optional values: Y/N'; - -CREATE OR REPLACE FUNCTION "update_partner_mapping_update_time"() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS "update_partner_mapping_update_time_trigger" ON "nexent"."partner_mapping_id_t"; -CREATE TRIGGER "update_partner_mapping_update_time_trigger" -BEFORE UPDATE ON "nexent"."partner_mapping_id_t" -FOR EACH ROW -EXECUTE FUNCTION "update_partner_mapping_update_time"(); \ No newline at end of file diff --git a/docker/sql/v1.7.2_0809_add_name_zh_to_ag_tenant_agent_t.sql b/docker/sql/v1.7.2_0809_add_name_zh_to_ag_tenant_agent_t.sql deleted file mode 100644 index 3b0b77c6c..000000000 --- a/docker/sql/v1.7.2_0809_add_name_zh_to_ag_tenant_agent_t.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS display_name VARCHAR(100); -COMMENT ON COLUMN nexent.ag_tenant_agent_t.display_name IS 'Agent展示名称'; \ No newline at end of file diff --git a/docker/sql/v1.7.2_0812_modify_model_record_t.sql b/docker/sql/v1.7.2_0812_modify_model_record_t.sql deleted file mode 100644 index 74acc8c30..000000000 --- a/docker/sql/v1.7.2_0812_modify_model_record_t.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE nexent.model_record_t -DROP COLUMN IF EXISTS is_deep_thinking; \ No newline at end of file diff --git a/docker/sql/v1.7.3.2_0902_add_model_name_to_knowledge_record_t.sql b/docker/sql/v1.7.3.2_0902_add_model_name_to_knowledge_record_t.sql deleted file mode 100644 index 3d0e30b27..000000000 --- a/docker/sql/v1.7.3.2_0902_add_model_name_to_knowledge_record_t.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Add model_name column to knowledge_record_t table, used to record the embedding model used by the knowledge base - --- Switch to nexent schema -SET search_path TO nexent; - --- Add model_name column -ALTER TABLE "knowledge_record_t" -ADD COLUMN IF NOT EXISTS "embedding_model_name" varchar(200) COLLATE "pg_catalog"."default"; - --- Add column comment -COMMENT ON COLUMN "knowledge_record_t"."embedding_model_name" IS 'Embedding model name, used to record the embedding model used by the knowledge base'; \ No newline at end of file diff --git a/docker/sql/v1.7.4.1_1011_add_origin_tool_name_to_ag_tool_info.sql b/docker/sql/v1.7.4.1_1011_add_origin_tool_name_to_ag_tool_info.sql deleted file mode 100644 index c312f83d2..000000000 --- a/docker/sql/v1.7.4.1_1011_add_origin_tool_name_to_ag_tool_info.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Add origin_name column to ag_tool_info_t table --- This field stores the original tool name before any transformations - -ALTER TABLE nexent.ag_tool_info_t -ADD COLUMN IF NOT EXISTS origin_name VARCHAR(100); - --- Add comment to document the purpose of this field -COMMENT ON COLUMN nexent.ag_tool_info_t.origin_name IS 'Original tool name before any transformations or mappings'; diff --git a/docker/sql/v1.7.4.1_1013_add_tool_group_to_ag_tool_info.sql b/docker/sql/v1.7.4.1_1013_add_tool_group_to_ag_tool_info.sql deleted file mode 100644 index b8cc4d294..000000000 --- a/docker/sql/v1.7.4.1_1013_add_tool_group_to_ag_tool_info.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Add category column to ag_tool_info_t table --- This field stores the tool category information (search, file, email, terminal) - -ALTER TABLE nexent.ag_tool_info_t -ADD COLUMN IF NOT EXISTS category VARCHAR(100); - --- Add comment to document the purpose of this field -COMMENT ON COLUMN nexent.ag_tool_info_t.category IS 'Tool category information'; diff --git a/docker/sql/v1.7.4_0928_add_model_id_to_ag_tenant_agent_t.sql b/docker/sql/v1.7.4_0928_add_model_id_to_ag_tenant_agent_t.sql deleted file mode 100644 index cfff187e0..000000000 --- a/docker/sql/v1.7.4_0928_add_model_id_to_ag_tenant_agent_t.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Add model_id column to ag_tenant_agent_t table and deprecate model_name field --- Date: 2024-09-28 --- Description: Add model_id field to ag_tenant_agent_t table and mark model_name as deprecated - --- Switch to the nexent schema -SET search_path TO nexent; - --- Add model_id column to ag_tenant_agent_t table -ALTER TABLE ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS model_id INTEGER; - --- Add comment for the new model_id column -COMMENT ON COLUMN ag_tenant_agent_t.model_id IS 'Model ID, foreign key reference to model_record_t.model_id'; - --- Update comment for model_name column to mark it as deprecated -COMMENT ON COLUMN ag_tenant_agent_t.model_name IS '[DEPRECATED] Name of the model used, use model_id instead'; - --- Optional: Add foreign key constraint (uncomment if needed) --- ALTER TABLE ag_tenant_agent_t --- ADD CONSTRAINT fk_ag_tenant_agent_model_id --- FOREIGN KEY (model_id) REFERENCES model_record_t(model_id); diff --git a/docker/sql/v1.7.5.1_1028_add_chunk_size_to_model_record_t.sql b/docker/sql/v1.7.5.1_1028_add_chunk_size_to_model_record_t.sql deleted file mode 100644 index 4fa08dc0f..000000000 --- a/docker/sql/v1.7.5.1_1028_add_chunk_size_to_model_record_t.sql +++ /dev/null @@ -1,7 +0,0 @@ -ALTER TABLE nexent.model_record_t -ADD COLUMN IF NOT EXISTS expected_chunk_size INT4, -ADD COLUMN IF NOT EXISTS maximum_chunk_size INT4; - -COMMENT ON COLUMN nexent.model_record_t.expected_chunk_size IS 'Expected chunk size for embedding models, used during document chunking'; -COMMENT ON COLUMN nexent.model_record_t.maximum_chunk_size IS 'Maximum chunk size for embedding models, used during document chunking'; - diff --git a/docker/sql/v1.7.5_1024_add_business_logic_model_fields.sql b/docker/sql/v1.7.5_1024_add_business_logic_model_fields.sql deleted file mode 100644 index ff1a7673c..000000000 --- a/docker/sql/v1.7.5_1024_add_business_logic_model_fields.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Add business_logic_model_name and business_logic_model_id fields to ag_tenant_agent_t table --- These fields store the LLM model used for generating business logic prompts - -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS business_logic_model_name VARCHAR(100); - -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS business_logic_model_id INTEGER; - -COMMENT ON COLUMN nexent.ag_tenant_agent_t.business_logic_model_name IS 'Model name used for business logic prompt generation'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.business_logic_model_id IS 'Model ID used for business logic prompt generation, foreign key reference to model_record_t.model_id'; - diff --git a/docker/sql/v1.7.5_1024_alter_tenant_config_t_config_value.sql b/docker/sql/v1.7.5_1024_alter_tenant_config_t_config_value.sql deleted file mode 100644 index 163cb7ea8..000000000 --- a/docker/sql/v1.7.5_1024_alter_tenant_config_t_config_value.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE nexent.tenant_config_t ALTER COLUMN config_value TYPE TEXT; \ No newline at end of file diff --git a/docker/sql/v1.7.7_1129_add_ssl_verify_to_model_record_t.sql b/docker/sql/v1.7.7_1129_add_ssl_verify_to_model_record_t.sql deleted file mode 100644 index 5eec1f92c..000000000 --- a/docker/sql/v1.7.7_1129_add_ssl_verify_to_model_record_t.sql +++ /dev/null @@ -1,5 +0,0 @@ -ALTER TABLE nexent.model_record_t -ADD COLUMN IF NOT EXISTS ssl_verify BOOLEAN DEFAULT TRUE; - -COMMENT ON COLUMN nexent.model_record_t.ssl_verify IS 'Whether to verify SSL certificates when connecting to this model API. Default is true. Set to false for local services without SSL support.'; - diff --git a/docker/sql/v1.7.8_1204_add_knowledge_name_to_knowledge_record_t.sql b/docker/sql/v1.7.8_1204_add_knowledge_name_to_knowledge_record_t.sql deleted file mode 100644 index 4e889bb0e..000000000 --- a/docker/sql/v1.7.8_1204_add_knowledge_name_to_knowledge_record_t.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Add knowledge_name column if it does not exist -ALTER TABLE nexent.knowledge_record_t -ADD COLUMN IF NOT EXISTS knowledge_name varchar(100) COLLATE "pg_catalog"."default"; - -COMMENT ON COLUMN nexent.knowledge_record_t.knowledge_name IS 'User-facing knowledge base name (display name), mapped to internal index_name'; -COMMENT ON COLUMN nexent.knowledge_record_t.index_name IS 'Internal Elasticsearch index name'; - --- Backfill existing records: for legacy data, use index_name as knowledge_name -UPDATE nexent.knowledge_record_t -SET knowledge_name = index_name -WHERE knowledge_name IS NULL; - - --- Add chunk_batch column in model_record_t table -ALTER TABLE nexent.model_record_t -ADD COLUMN IF NOT EXISTS chunk_batch INT4; - -COMMENT ON COLUMN nexent.model_record_t.chunk_batch IS 'Batch size for concurrent embedding requests during document chunking'; \ No newline at end of file diff --git a/docker/sql/v1.7.8_add_author_to_ag_tenant_agent_t.sql b/docker/sql/v1.7.8_add_author_to_ag_tenant_agent_t.sql deleted file mode 100644 index 4ac134624..000000000 --- a/docker/sql/v1.7.8_add_author_to_ag_tenant_agent_t.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Add author column to ag_tenant_agent_t table --- This migration adds the author field to support agent author information - --- Add author column with default NULL value for backward compatibility -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS author VARCHAR(100); - --- Add comment to the column -COMMENT ON COLUMN nexent.ag_tenant_agent_t.author IS 'Agent author'; - diff --git a/docker/sql/v1.7.9.2_1226_add_invitation_and_group_system.sql b/docker/sql/v1.7.9.2_1226_add_invitation_and_group_system.sql deleted file mode 100644 index 75c471404..000000000 --- a/docker/sql/v1.7.9.2_1226_add_invitation_and_group_system.sql +++ /dev/null @@ -1,360 +0,0 @@ --- Add invitation code and group management system --- This migration adds invitation codes, groups, and permission management features - --- 1. Create tenant_invitation_code_t table for invitation codes -CREATE TABLE IF NOT EXISTS nexent.tenant_invitation_code_t ( - invitation_id SERIAL PRIMARY KEY, - tenant_id VARCHAR(100) NOT NULL, - invitation_code VARCHAR(100) NOT NULL, - group_ids VARCHAR, -- int4 list - capacity INT4 NOT NULL DEFAULT 1, - expiry_date TIMESTAMP(6) WITHOUT TIME ZONE, - status VARCHAR(30) NOT NULL, - code_type VARCHAR(30) NOT NULL, - create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Add comments for tenant_invitation_code_t table -COMMENT ON TABLE nexent.tenant_invitation_code_t IS 'Tenant invitation code information table'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.invitation_id IS 'Invitation ID, primary key'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.tenant_id IS 'Tenant ID, foreign key'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.invitation_code IS 'Invitation code'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.group_ids IS 'Associated group IDs list'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.capacity IS 'Invitation code capacity'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.expiry_date IS 'Invitation code expiry date'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.status IS 'Invitation code status: IN_USE, EXPIRE, DISABLE, RUN_OUT'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.code_type IS 'Invitation code type: ADMIN_INVITE, DEV_INVITE, USER_INVITE'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.create_time IS 'Create time'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.created_by IS 'Created by'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.updated_by IS 'Updated by'; -COMMENT ON COLUMN nexent.tenant_invitation_code_t.delete_flag IS 'Delete flag, Y/N'; - --- 2. Create tenant_invitation_record_t table for invitation usage records -CREATE TABLE IF NOT EXISTS nexent.tenant_invitation_record_t ( - invitation_record_id SERIAL PRIMARY KEY, - invitation_id INT4 NOT NULL, - user_id VARCHAR(100) NOT NULL, - create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Add comments for tenant_invitation_record_t table -COMMENT ON TABLE nexent.tenant_invitation_record_t IS 'Tenant invitation record table'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.invitation_record_id IS 'Invitation record ID, primary key'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.invitation_id IS 'Invitation ID, foreign key'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.create_time IS 'Create time'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.created_by IS 'Created by'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.updated_by IS 'Updated by'; -COMMENT ON COLUMN nexent.tenant_invitation_record_t.delete_flag IS 'Delete flag, Y/N'; - --- 3. Create tenant_group_info_t table for group information -CREATE TABLE IF NOT EXISTS nexent.tenant_group_info_t ( - group_id SERIAL PRIMARY KEY, - tenant_id VARCHAR(100) NOT NULL, - group_name VARCHAR(100) NOT NULL, - group_description VARCHAR(500), - create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Add comments for tenant_group_info_t table -COMMENT ON TABLE nexent.tenant_group_info_t IS 'Tenant group information table'; -COMMENT ON COLUMN nexent.tenant_group_info_t.group_id IS 'Group ID, primary key'; -COMMENT ON COLUMN nexent.tenant_group_info_t.tenant_id IS 'Tenant ID, foreign key'; -COMMENT ON COLUMN nexent.tenant_group_info_t.group_name IS 'Group name'; -COMMENT ON COLUMN nexent.tenant_group_info_t.group_description IS 'Group description'; -COMMENT ON COLUMN nexent.tenant_group_info_t.create_time IS 'Create time'; -COMMENT ON COLUMN nexent.tenant_group_info_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.tenant_group_info_t.created_by IS 'Created by'; -COMMENT ON COLUMN nexent.tenant_group_info_t.updated_by IS 'Updated by'; -COMMENT ON COLUMN nexent.tenant_group_info_t.delete_flag IS 'Delete flag, Y/N'; - --- 4. Create tenant_group_user_t table for group user membership -CREATE TABLE IF NOT EXISTS nexent.tenant_group_user_t ( - group_user_id SERIAL PRIMARY KEY, - group_id INT4 NOT NULL, - user_id VARCHAR(100) NOT NULL, - create_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP(6) WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Add comments for tenant_group_user_t table -COMMENT ON TABLE nexent.tenant_group_user_t IS 'Tenant group user membership table'; -COMMENT ON COLUMN nexent.tenant_group_user_t.group_user_id IS 'Group user ID, primary key'; -COMMENT ON COLUMN nexent.tenant_group_user_t.group_id IS 'Group ID, foreign key'; -COMMENT ON COLUMN nexent.tenant_group_user_t.user_id IS 'User ID, foreign key'; -COMMENT ON COLUMN nexent.tenant_group_user_t.create_time IS 'Create time'; -COMMENT ON COLUMN nexent.tenant_group_user_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.tenant_group_user_t.created_by IS 'Created by'; -COMMENT ON COLUMN nexent.tenant_group_user_t.updated_by IS 'Updated by'; -COMMENT ON COLUMN nexent.tenant_group_user_t.delete_flag IS 'Delete flag, Y/N'; - --- 5. Add fields to user_tenant_t table -ALTER TABLE nexent.user_tenant_t -ADD COLUMN IF NOT EXISTS user_role VARCHAR(30); - --- Add comments for new fields in user_tenant_t table -COMMENT ON COLUMN nexent.user_tenant_t.user_role IS 'User role: SU, ADMIN, DEV, USER'; - --- 6. Create role_permission_t table for role permissions -CREATE TABLE IF NOT EXISTS nexent.role_permission_t ( - role_permission_id SERIAL PRIMARY KEY, - user_role VARCHAR(30) NOT NULL, - permission_category VARCHAR(30), - permission_type VARCHAR(30), - permission_subtype VARCHAR(30) -); - --- Add comments for role_permission_t table -COMMENT ON TABLE nexent.role_permission_t IS 'Role permission configuration table'; -COMMENT ON COLUMN nexent.role_permission_t.role_permission_id IS 'Role permission ID, primary key'; -COMMENT ON COLUMN nexent.role_permission_t.user_role IS 'User role: SU, ADMIN, DEV, USER'; -COMMENT ON COLUMN nexent.role_permission_t.permission_category IS 'Permission category'; -COMMENT ON COLUMN nexent.role_permission_t.permission_type IS 'Permission type'; -COMMENT ON COLUMN nexent.role_permission_t.permission_subtype IS 'Permission subtype'; - --- 7. Add fields to knowledge_record_t table -ALTER TABLE nexent.knowledge_record_t -ADD COLUMN IF NOT EXISTS group_ids VARCHAR, -- int4 list -ADD COLUMN IF NOT EXISTS ingroup_permission VARCHAR(30); - --- Add comments for new fields in knowledge_record_t table -COMMENT ON COLUMN nexent.knowledge_record_t.group_ids IS 'Knowledge base group IDs list'; -COMMENT ON COLUMN nexent.knowledge_record_t.ingroup_permission IS 'In-group permission: EDIT, READ_ONLY, PRIVATE'; - --- 8. Add fields to ag_tenant_agent_t table -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS group_ids VARCHAR; -- int4 list - --- Add comments for new fields in ag_tenant_agent_t table -COMMENT ON COLUMN nexent.ag_tenant_agent_t.group_ids IS 'Agent group IDs list'; - --- 9. Insert role permission data -INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES -(1, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(2, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(3, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(4, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), -(5, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(6, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(7, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(8, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(9, 'SU', 'RESOURCE', 'AGENT', 'READ'), -(10, 'SU', 'RESOURCE', 'AGENT', 'DELETE'), -(11, 'SU', 'RESOURCE', 'KB', 'READ'), -(12, 'SU', 'RESOURCE', 'KB', 'DELETE'), -(13, 'SU', 'RESOURCE', 'KB.GROUPS', 'READ'), -(14, 'SU', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(15, 'SU', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(16, 'SU', 'RESOURCE', 'USER.ROLE', 'READ'), -(17, 'SU', 'RESOURCE', 'USER.ROLE', 'UPDATE'), -(18, 'SU', 'RESOURCE', 'USER.ROLE', 'DELETE'), -(19, 'SU', 'RESOURCE', 'MCP', 'READ'), -(20, 'SU', 'RESOURCE', 'MCP', 'DELETE'), -(21, 'SU', 'RESOURCE', 'MEM.SETTING', 'READ'), -(22, 'SU', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(23, 'SU', 'RESOURCE', 'MEM.AGENT', 'READ'), -(24, 'SU', 'RESOURCE', 'MEM.AGENT', 'DELETE'), -(25, 'SU', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(26, 'SU', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(27, 'SU', 'RESOURCE', 'MODEL', 'CREATE'), -(28, 'SU', 'RESOURCE', 'MODEL', 'READ'), -(29, 'SU', 'RESOURCE', 'MODEL', 'UPDATE'), -(30, 'SU', 'RESOURCE', 'MODEL', 'DELETE'), -(31, 'SU', 'RESOURCE', 'TENANT', 'CREATE'), -(32, 'SU', 'RESOURCE', 'TENANT', 'READ'), -(33, 'SU', 'RESOURCE', 'TENANT', 'UPDATE'), -(34, 'SU', 'RESOURCE', 'TENANT', 'DELETE'), -(35, 'SU', 'RESOURCE', 'TENANT.INFO', 'READ'), -(36, 'SU', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), -(37, 'SU', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), -(38, 'SU', 'RESOURCE', 'TENANT.INVITE', 'READ'), -(39, 'SU', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), -(40, 'SU', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), -(41, 'SU', 'RESOURCE', 'GROUP', 'CREATE'), -(42, 'SU', 'RESOURCE', 'GROUP', 'READ'), -(43, 'SU', 'RESOURCE', 'GROUP', 'UPDATE'), -(44, 'SU', 'RESOURCE', 'GROUP', 'DELETE'), -(45, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(46, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(47, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), -(48, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(49, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), -(50, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), -(51, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(52, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), -(53, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(54, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(55, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(56, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(57, 'ADMIN', 'RESOURCE', 'AGENT', 'CREATE'), -(58, 'ADMIN', 'RESOURCE', 'AGENT', 'READ'), -(59, 'ADMIN', 'RESOURCE', 'AGENT', 'UPDATE'), -(60, 'ADMIN', 'RESOURCE', 'AGENT', 'DELETE'), -(61, 'ADMIN', 'RESOURCE', 'KB', 'CREATE'), -(62, 'ADMIN', 'RESOURCE', 'KB', 'READ'), -(63, 'ADMIN', 'RESOURCE', 'KB', 'UPDATE'), -(64, 'ADMIN', 'RESOURCE', 'KB', 'DELETE'), -(65, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'READ'), -(66, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(67, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(68, 'ADMIN', 'RESOURCE', 'USER.ROLE', 'READ'), -(69, 'ADMIN', 'RESOURCE', 'MCP', 'CREATE'), -(70, 'ADMIN', 'RESOURCE', 'MCP', 'READ'), -(71, 'ADMIN', 'RESOURCE', 'MCP', 'UPDATE'), -(72, 'ADMIN', 'RESOURCE', 'MCP', 'DELETE'), -(73, 'ADMIN', 'RESOURCE', 'MEM.SETTING', 'READ'), -(74, 'ADMIN', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(75, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'CREATE'), -(76, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'READ'), -(77, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'DELETE'), -(78, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(79, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(80, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(81, 'ADMIN', 'RESOURCE', 'MODEL', 'CREATE'), -(82, 'ADMIN', 'RESOURCE', 'MODEL', 'READ'), -(83, 'ADMIN', 'RESOURCE', 'MODEL', 'UPDATE'), -(84, 'ADMIN', 'RESOURCE', 'MODEL', 'DELETE'), -(85, 'ADMIN', 'RESOURCE', 'TENANT.INFO', 'READ'), -(86, 'ADMIN', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), -(87, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), -(88, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'READ'), -(89, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), -(90, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), -(91, 'ADMIN', 'RESOURCE', 'GROUP', 'CREATE'), -(92, 'ADMIN', 'RESOURCE', 'GROUP', 'READ'), -(93, 'ADMIN', 'RESOURCE', 'GROUP', 'UPDATE'), -(94, 'ADMIN', 'RESOURCE', 'GROUP', 'DELETE'), -(95, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(96, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(97, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), -(98, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(99, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), -(100, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), -(101, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(102, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), -(103, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(104, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(105, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(106, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(107, 'DEV', 'RESOURCE', 'AGENT', 'CREATE'), -(108, 'DEV', 'RESOURCE', 'AGENT', 'READ'), -(109, 'DEV', 'RESOURCE', 'AGENT', 'UPDATE'), -(110, 'DEV', 'RESOURCE', 'AGENT', 'DELETE'), -(111, 'DEV', 'RESOURCE', 'KB', 'CREATE'), -(112, 'DEV', 'RESOURCE', 'KB', 'READ'), -(113, 'DEV', 'RESOURCE', 'KB', 'UPDATE'), -(114, 'DEV', 'RESOURCE', 'KB', 'DELETE'), -(115, 'DEV', 'RESOURCE', 'KB.GROUPS', 'READ'), -(116, 'DEV', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(117, 'DEV', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(118, 'DEV', 'RESOURCE', 'USER.ROLE', 'READ'), -(119, 'DEV', 'RESOURCE', 'MCP', 'CREATE'), -(120, 'DEV', 'RESOURCE', 'MCP', 'READ'), -(121, 'DEV', 'RESOURCE', 'MCP', 'UPDATE'), -(122, 'DEV', 'RESOURCE', 'MCP', 'DELETE'), -(123, 'DEV', 'RESOURCE', 'MEM.SETTING', 'READ'), -(124, 'DEV', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(125, 'DEV', 'RESOURCE', 'MEM.AGENT', 'READ'), -(126, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(127, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(128, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(129, 'DEV', 'RESOURCE', 'MODEL', 'READ'), -(130, 'DEV', 'RESOURCE', 'TENANT.INFO', 'READ'), -(131, 'DEV', 'RESOURCE', 'GROUP', 'READ'), -(132, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(133, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(134, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(135, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(136, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(137, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(138, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(139, 'USER', 'RESOURCE', 'AGENT', 'READ'), -(140, 'USER', 'RESOURCE', 'KB', 'CREATE'), -(141, 'USER', 'RESOURCE', 'KB', 'READ'), -(142, 'USER', 'RESOURCE', 'KB', 'UPDATE'), -(143, 'USER', 'RESOURCE', 'KB', 'DELETE'), -(144, 'USER', 'RESOURCE', 'KB.GROUPS', 'READ'), -(145, 'USER', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(146, 'USER', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(147, 'USER', 'RESOURCE', 'USER.ROLE', 'READ'), -(148, 'USER', 'RESOURCE', 'MCP', 'CREATE'), -(149, 'USER', 'RESOURCE', 'MCP', 'READ'), -(150, 'USER', 'RESOURCE', 'MCP', 'UPDATE'), -(151, 'USER', 'RESOURCE', 'MCP', 'DELETE'), -(152, 'USER', 'RESOURCE', 'MEM.SETTING', 'READ'), -(153, 'USER', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(154, 'USER', 'RESOURCE', 'MEM.AGENT', 'READ'), -(155, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(156, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(157, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(158, 'USER', 'RESOURCE', 'MODEL', 'READ'), -(159, 'USER', 'RESOURCE', 'TENANT.INFO', 'READ'), -(160, 'USER', 'RESOURCE', 'GROUP', 'READ'), -(161, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(162, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(163, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), -(164, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(165, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), -(166, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), -(167, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(168, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), -(169, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(170, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(171, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(172, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(173, 'SPEED', 'RESOURCE', 'AGENT', 'CREATE'), -(174, 'SPEED', 'RESOURCE', 'AGENT', 'READ'), -(175, 'SPEED', 'RESOURCE', 'AGENT', 'UPDATE'), -(176, 'SPEED', 'RESOURCE', 'AGENT', 'DELETE'), -(177, 'SPEED', 'RESOURCE', 'KB', 'CREATE'), -(178, 'SPEED', 'RESOURCE', 'KB', 'READ'), -(179, 'SPEED', 'RESOURCE', 'KB', 'UPDATE'), -(180, 'SPEED', 'RESOURCE', 'KB', 'DELETE'), -(181, 'SPEED', 'RESOURCE', 'KB.GROUPS', 'READ'), -(182, 'SPEED', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(183, 'SPEED', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(184, 'SPEED', 'RESOURCE', 'USER.ROLE', 'READ'), -(185, 'SPEED', 'RESOURCE', 'MCP', 'CREATE'), -(186, 'SPEED', 'RESOURCE', 'MCP', 'READ'), -(187, 'SPEED', 'RESOURCE', 'MCP', 'UPDATE'), -(188, 'SPEED', 'RESOURCE', 'MCP', 'DELETE'), -(189, 'SPEED', 'RESOURCE', 'MEM.SETTING', 'READ'), -(190, 'SPEED', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(191, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'CREATE'), -(192, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'READ'), -(193, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'DELETE'), -(194, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(195, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(196, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(197, 'SPEED', 'RESOURCE', 'MODEL', 'CREATE'), -(198, 'SPEED', 'RESOURCE', 'MODEL', 'READ'), -(199, 'SPEED', 'RESOURCE', 'MODEL', 'UPDATE'), -(200, 'SPEED', 'RESOURCE', 'MODEL', 'DELETE'), -(201, 'SPEED', 'RESOURCE', 'TENANT.INFO', 'READ'), -(202, 'SPEED', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), -(203, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), -(204, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'READ'), -(205, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), -(206, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), -(207, 'SPEED', 'RESOURCE', 'GROUP', 'CREATE'), -(208, 'SPEED', 'RESOURCE', 'GROUP', 'READ'), -(209, 'SPEED', 'RESOURCE', 'GROUP', 'UPDATE'), -(210, 'SPEED', 'RESOURCE', 'GROUP', 'DELETE') -ON CONFLICT (role_permission_id) DO NOTHING; diff --git a/docker/sql/v1.7.9.3_0122_add_is_new_to_ag_tenant_agent_t.sql b/docker/sql/v1.7.9.3_0122_add_is_new_to_ag_tenant_agent_t.sql deleted file mode 100644 index 2e8e538c4..000000000 --- a/docker/sql/v1.7.9.3_0122_add_is_new_to_ag_tenant_agent_t.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Add is_new column to ag_tenant_agent_t table for new agent marking --- This migration adds a field to track whether an agent is marked as new for users - --- Add is_new column with default value false -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS is_new BOOLEAN DEFAULT FALSE; - --- Add comment for the new column -COMMENT ON COLUMN nexent.ag_tenant_agent_t.is_new IS 'Whether this agent is marked as new for the user'; - --- Create index for performance on is_new queries -CREATE INDEX IF NOT EXISTS idx_ag_tenant_agent_t_is_new -ON nexent.ag_tenant_agent_t (tenant_id, is_new) -WHERE delete_flag = 'N'; - - diff --git a/docker/sql/v1.7.9.3_0123_add_speed_user_tenant_t.sql b/docker/sql/v1.7.9.3_0123_add_speed_user_tenant_t.sql deleted file mode 100644 index e0d5b3ce6..000000000 --- a/docker/sql/v1.7.9.3_0123_add_speed_user_tenant_t.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Add user_email column to user_tenant_t table -ALTER TABLE nexent.user_tenant_t -ADD COLUMN IF NOT EXISTS user_email VARCHAR(255); - --- Add comment to the new column -COMMENT ON COLUMN nexent.user_tenant_t.user_email IS 'User email address'; - -INSERT INTO nexent.user_tenant_t (user_id, tenant_id, user_role, user_email, created_by, updated_by) -VALUES ('user_id', 'tenant_id', 'SPEED', NULL, 'system', 'system') -ON CONFLICT (user_id, tenant_id) DO NOTHING; diff --git a/docker/sql/v1.7.9_1219_add_container_id_to_mcp_record_t.sql b/docker/sql/v1.7.9_1219_add_container_id_to_mcp_record_t.sql deleted file mode 100644 index 553f484e6..000000000 --- a/docker/sql/v1.7.9_1219_add_container_id_to_mcp_record_t.sql +++ /dev/null @@ -1,6 +0,0 @@ -ALTER TABLE nexent.mcp_record_t -ADD COLUMN IF NOT EXISTS container_id VARCHAR(200); - -COMMENT ON COLUMN nexent.mcp_record_t.container_id IS 'Docker container ID for MCP service, NULL for non-containerized MCP'; - - diff --git a/docker/sql/v1.8.0.1_0224_init_agent_id_seq.sql b/docker/sql/v1.8.0.1_0224_init_agent_id_seq.sql deleted file mode 100644 index 67b6bd091..000000000 --- a/docker/sql/v1.8.0.1_0224_init_agent_id_seq.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE SEQUENCE IF NOT EXISTS "nexent"."ag_tenant_agent_t_agent_id_seq" -INCREMENT 1 -MINVALUE 1 -MAXVALUE 2147483647 -START 1 -CACHE 1; \ No newline at end of file diff --git a/docker/sql/v1.8.0.1_0225_delete_empty_tenant.sql b/docker/sql/v1.8.0.1_0225_delete_empty_tenant.sql deleted file mode 100644 index 0c0bb8a0b..000000000 --- a/docker/sql/v1.8.0.1_0225_delete_empty_tenant.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Delete erroneous tenant with empty tenant_id and all related data --- This script removes records where tenant_id is empty string from tenant_config_t and tenant_group_info_t - --- 1. Force delete all records in tenant_config_t where tenant_id is empty string -DELETE FROM nexent.tenant_config_t -WHERE tenant_id = ''; - --- 2. Force delete all records in tenant_group_info_t where tenant_id is empty string -DELETE FROM nexent.tenant_group_info_t -WHERE tenant_id = ''; diff --git a/docker/sql/v1.8.0.1_0226_add_authorization_token_to_mcp_record_t.sql b/docker/sql/v1.8.0.1_0226_add_authorization_token_to_mcp_record_t.sql deleted file mode 100644 index f9ce4ba73..000000000 --- a/docker/sql/v1.8.0.1_0226_add_authorization_token_to_mcp_record_t.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Migration: Add authorization_token column to mcp_record_t table --- Date: 2025-03-01 --- Description: Add authorization_token field to support MCP server authentication - --- Add authorization_token column to mcp_record_t table -ALTER TABLE nexent.mcp_record_t -ADD COLUMN IF NOT EXISTS authorization_token VARCHAR(500) DEFAULT NULL; - --- Add comment to the column -COMMENT ON COLUMN nexent.mcp_record_t.authorization_token IS 'Authorization token for MCP server authentication (e.g., Bearer token)'; diff --git a/docker/sql/v1.8.0.2_0227_add_ingroup_permission_to_ag_tenant_agent_t.sql b/docker/sql/v1.8.0.2_0227_add_ingroup_permission_to_ag_tenant_agent_t.sql deleted file mode 100644 index 38ae17814..000000000 --- a/docker/sql/v1.8.0.2_0227_add_ingroup_permission_to_ag_tenant_agent_t.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Migration: Add ingroup_permission column to ag_tenant_agent_t table --- Date: 2025-03-02 --- Description: Add ingroup_permission field to support in-group permission control for agents - --- Add ingroup_permission column to ag_tenant_agent_t table -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS ingroup_permission VARCHAR(30) DEFAULT NULL; - --- Add comment to the column -COMMENT ON COLUMN nexent.ag_tenant_agent_t.ingroup_permission IS 'In-group permission: EDIT, READ_ONLY, PRIVATE'; diff --git a/docker/sql/v1.8.0.2_0302_add_tool_instance_id_seq_and_agent_relation_id_seq.sql b/docker/sql/v1.8.0.2_0302_add_tool_instance_id_seq_and_agent_relation_id_seq.sql deleted file mode 100644 index 06fde6435..000000000 --- a/docker/sql/v1.8.0.2_0302_add_tool_instance_id_seq_and_agent_relation_id_seq.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Step 1: Create sequence for auto-increment -CREATE SEQUENCE IF NOT EXISTS "nexent"."ag_tool_instance_t_tool_instance_id_seq" -INCREMENT 1 -MINVALUE 1 -MAXVALUE 2147483647 -START 1 -CACHE 1; - -CREATE SEQUENCE IF NOT EXISTS "nexent"."ag_agent_relation_t_relation_id_seq" -INCREMENT 1 -MINVALUE 1 -MAXVALUE 2147483647 -START 1 -CACHE 1; diff --git a/docker/sql/v1.8.0_0204_init_tenant_group.sql b/docker/sql/v1.8.0_0204_init_tenant_group.sql deleted file mode 100644 index fde946cb9..000000000 --- a/docker/sql/v1.8.0_0204_init_tenant_group.sql +++ /dev/null @@ -1,76 +0,0 @@ --- Initialize tenant group and default configuration for existing tenants --- This migration adds default group and basic config for tenants that lack them --- Trigger condition: tenant has no TENANT_ID config_key in tenant_config_t - -DO $$ -DECLARE - target_tenant_id VARCHAR(100); - new_group_id INTEGER; -BEGIN - -- Loop through each distinct tenant_id from user_tenant_t - FOR target_tenant_id IN - SELECT DISTINCT tenant_id - FROM nexent.user_tenant_t - WHERE tenant_id IS NOT NULL - LOOP - -- Check if tenant already has TENANT_ID config_key - IF NOT EXISTS ( - SELECT 1 FROM nexent.tenant_config_t - WHERE tenant_id = target_tenant_id - AND config_key = 'TENANT_ID' - AND delete_flag = 'N' - ) THEN - -- Insert TENANT_ID config - INSERT INTO nexent.tenant_config_t ( - tenant_id, user_id, value_type, config_key, config_value, - create_time, update_time, created_by, updated_by, delete_flag - ) VALUES ( - target_tenant_id, NULL, 'single', 'TENANT_ID', target_tenant_id, - NOW(), NOW(), 'system', 'system', 'N' - ); - - -- Insert TENANT_NAME config if not exists - IF NOT EXISTS ( - SELECT 1 FROM nexent.tenant_config_t - WHERE tenant_id = target_tenant_id - AND config_key = 'TENANT_NAME' - AND delete_flag = 'N' - ) THEN - INSERT INTO nexent.tenant_config_t ( - tenant_id, user_id, value_type, config_key, config_value, - create_time, update_time, created_by, updated_by, delete_flag - ) VALUES ( - target_tenant_id, NULL, 'single', 'TENANT_NAME', 'Unnamed Tenant', - NOW(), NOW(), 'system', 'system', 'N' - ); - END IF; - - -- Check if tenant already has a group - IF NOT EXISTS ( - SELECT 1 FROM nexent.tenant_group_info_t - WHERE tenant_id = target_tenant_id - AND delete_flag = 'N' - ) THEN - -- Insert default group - INSERT INTO nexent.tenant_group_info_t ( - tenant_id, group_name, group_description, - create_time, update_time, created_by, updated_by, delete_flag - ) VALUES ( - target_tenant_id, 'Default Group', 'Default group for tenant', - NOW(), NOW(), 'system', 'system', 'N' - ) RETURNING group_id INTO new_group_id; - - -- Insert DEFAULT_GROUP_ID config - IF new_group_id IS NOT NULL THEN - INSERT INTO nexent.tenant_config_t ( - tenant_id, user_id, value_type, config_key, config_value, - create_time, update_time, created_by, updated_by, delete_flag - ) VALUES ( - target_tenant_id, NULL, 'single', 'DEFAULT_GROUP_ID', new_group_id::VARCHAR, - NOW(), NOW(), 'system', 'system', 'N' - ); - END IF; - END IF; - END IF; - END LOOP; -END $$; diff --git a/docker/sql/v1.8.0_0206_add_ag_tenant_agent_version_t .sql b/docker/sql/v1.8.0_0206_add_ag_tenant_agent_version_t .sql deleted file mode 100644 index 40fc22df0..000000000 --- a/docker/sql/v1.8.0_0206_add_ag_tenant_agent_version_t .sql +++ /dev/null @@ -1,84 +0,0 @@ --- 步骤 1：添加 nullable 的 version_no 字段（不设默认值，让显式赋值） -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS version_no INTEGER NULL; - -ALTER TABLE nexent.ag_tool_instance_t -ADD COLUMN IF NOT EXISTS version_no INTEGER NULL; - -ALTER TABLE nexent.ag_agent_relation_t -ADD COLUMN IF NOT EXISTS version_no INTEGER NULL; - --- 步骤 2：更新所有历史数据的 version_no 为 0 -UPDATE nexent.ag_tenant_agent_t SET version_no = 0 WHERE version_no IS NULL; -UPDATE nexent.ag_tool_instance_t SET version_no = 0 WHERE version_no IS NULL; -UPDATE nexent.ag_agent_relation_t SET version_no = 0 WHERE version_no IS NULL; - --- 步骤 3：将字段设为 NOT NULL，并设置默认值 0 -ALTER TABLE nexent.ag_tenant_agent_t ALTER COLUMN version_no SET NOT NULL; -ALTER TABLE nexent.ag_tenant_agent_t ALTER COLUMN version_no SET DEFAULT 0; - -ALTER TABLE nexent.ag_tool_instance_t ALTER COLUMN version_no SET NOT NULL; -ALTER TABLE nexent.ag_tool_instance_t ALTER COLUMN version_no SET DEFAULT 0; - -ALTER TABLE nexent.ag_agent_relation_t ALTER COLUMN version_no SET NOT NULL; -ALTER TABLE nexent.ag_agent_relation_t ALTER COLUMN version_no SET DEFAULT 0; - --- 步骤 4：为 ag_tenant_agent_t 添加 current_version_no 字段 -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS current_version_no INTEGER NULL; - --- 步骤5：修改主键 -ALTER TABLE nexent.ag_tenant_agent_t DROP CONSTRAINT ag_tenant_agent_t_pkey; -ALTER TABLE nexent.ag_tenant_agent_t ADD CONSTRAINT ag_tenant_agent_t_pkey PRIMARY KEY (agent_id, version_no); - -ALTER TABLE nexent.ag_tool_instance_t DROP CONSTRAINT ag_tool_instance_t_pkey; -ALTER TABLE nexent.ag_tool_instance_t ADD CONSTRAINT ag_tool_instance_t_pkey PRIMARY KEY (tool_instance_id, version_no); - -ALTER TABLE nexent.ag_agent_relation_t DROP CONSTRAINT ag_agent_relation_t_pkey; -ALTER TABLE nexent.ag_agent_relation_t ADD CONSTRAINT ag_agent_relation_t_pkey PRIMARY KEY (relation_id, version_no); - --- 步骤6：新增agent版本管理表 -CREATE TABLE IF NOT EXISTS nexent.ag_tenant_agent_version_t ( - id BIGSERIAL PRIMARY KEY, - tenant_id VARCHAR(100) NOT NULL, - agent_id INTEGER NOT NULL, - version_no INTEGER NOT NULL, - version_name VARCHAR(100), -- 用户自定义版本名称 - release_note TEXT, -- 发布备注 - - source_version_no INTEGER NULL, -- 来源版本号（回滚时记录） - source_type VARCHAR(30) NULL, -- 来源类型：NORMAL(正常发布) / ROLLBACK(回滚产生) - - status VARCHAR(30) DEFAULT 'RELEASED', -- 版本状态：RELEASED / DISABLED / ARCHIVED - - created_by VARCHAR(100) NOT NULL, - create_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - updated_by VARCHAR(100), - update_time TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.ag_tenant_agent_version_t OWNER TO "root"; - --- 步骤 7：添加COMMENT -COMMENT ON COLUMN nexent.ag_tenant_agent_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.current_version_no IS 'Current published version number. NULL means no version published yet'; -COMMENT ON COLUMN nexent.ag_tool_instance_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; -COMMENT ON COLUMN nexent.ag_agent_relation_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; - -COMMENT ON TABLE nexent.ag_tenant_agent_version_t IS 'Agent version metadata table. Stores version info, release notes, and version lineage.'; - -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.id IS 'Primary key, auto-increment'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.agent_id IS 'Agent ID'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.version_no IS 'Version number, starts from 1. Does not include 0 (draft)'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.version_name IS 'User-defined version name for display (e.g., "Stable v2.1", "Hotfix-001"). NULL means use version_no as display.'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.release_note IS 'Release notes / publish remarks'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.source_version_no IS 'Source version number. If this version is a rollback, record the source version number.'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.source_type IS 'Source type: NORMAL (normal publish) / ROLLBACK (rollback and republish).'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.status IS 'Version status: RELEASED / DISABLED / ARCHIVED'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.created_by IS 'User who published this version'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.create_time IS 'Version creation timestamp'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.updated_by IS 'Last user who updated this version'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.update_time IS 'Last update timestamp'; -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.delete_flag IS 'Soft delete flag: Y/N'; diff --git a/docker/sql/v1.8.0_0206_init_role_permission_t.sql b/docker/sql/v1.8.0_0206_init_role_permission_t.sql deleted file mode 100644 index 6b9409503..000000000 --- a/docker/sql/v1.8.0_0206_init_role_permission_t.sql +++ /dev/null @@ -1,186 +0,0 @@ -DELETE FROM nexent.role_permission_t; - -INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES -(1, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(2, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(3, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/tenant-resources'), -(4, 'SU', 'RESOURCE', 'AGENT', 'READ'), -(5, 'SU', 'RESOURCE', 'AGENT', 'DELETE'), -(6, 'SU', 'RESOURCE', 'KB', 'READ'), -(7, 'SU', 'RESOURCE', 'KB', 'DELETE'), -(8, 'SU', 'RESOURCE', 'KB.GROUPS', 'READ'), -(9, 'SU', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(10, 'SU', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(11, 'SU', 'RESOURCE', 'USER.ROLE', 'READ'), -(12, 'SU', 'RESOURCE', 'USER.ROLE', 'UPDATE'), -(13, 'SU', 'RESOURCE', 'USER.ROLE', 'DELETE'), -(14, 'SU', 'RESOURCE', 'MCP', 'READ'), -(15, 'SU', 'RESOURCE', 'MCP', 'DELETE'), -(16, 'SU', 'RESOURCE', 'MEM.SETTING', 'READ'), -(17, 'SU', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(18, 'SU', 'RESOURCE', 'MEM.AGENT', 'READ'), -(19, 'SU', 'RESOURCE', 'MEM.AGENT', 'DELETE'), -(20, 'SU', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(21, 'SU', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(22, 'SU', 'RESOURCE', 'MODEL', 'CREATE'), -(23, 'SU', 'RESOURCE', 'MODEL', 'READ'), -(24, 'SU', 'RESOURCE', 'MODEL', 'UPDATE'), -(25, 'SU', 'RESOURCE', 'MODEL', 'DELETE'), -(26, 'SU', 'RESOURCE', 'TENANT', 'CREATE'), -(27, 'SU', 'RESOURCE', 'TENANT', 'READ'), -(28, 'SU', 'RESOURCE', 'TENANT', 'UPDATE'), -(29, 'SU', 'RESOURCE', 'TENANT', 'DELETE'), -(30, 'SU', 'RESOURCE', 'TENANT.LIST', 'READ'), -(31, 'SU', 'RESOURCE', 'TENANT.INFO', 'READ'), -(32, 'SU', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), -(33, 'SU', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), -(34, 'SU', 'RESOURCE', 'TENANT.INVITE', 'READ'), -(35, 'SU', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), -(36, 'SU', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), -(37, 'SU', 'RESOURCE', 'GROUP', 'CREATE'), -(38, 'SU', 'RESOURCE', 'GROUP', 'READ'), -(39, 'SU', 'RESOURCE', 'GROUP', 'UPDATE'), -(40, 'SU', 'RESOURCE', 'GROUP', 'DELETE'), -(41, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(42, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(43, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), -(44, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(45, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), -(46, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), -(47, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(48, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), -(49, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(50, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(51, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(52, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(53, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/tenant-resources'), -(54, 'ADMIN', 'RESOURCE', 'AGENT', 'CREATE'), -(55, 'ADMIN', 'RESOURCE', 'AGENT', 'READ'), -(56, 'ADMIN', 'RESOURCE', 'AGENT', 'UPDATE'), -(57, 'ADMIN', 'RESOURCE', 'AGENT', 'DELETE'), -(58, 'ADMIN', 'RESOURCE', 'KB', 'CREATE'), -(59, 'ADMIN', 'RESOURCE', 'KB', 'READ'), -(60, 'ADMIN', 'RESOURCE', 'KB', 'UPDATE'), -(61, 'ADMIN', 'RESOURCE', 'KB', 'DELETE'), -(62, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'READ'), -(63, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(64, 'ADMIN', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(65, 'ADMIN', 'RESOURCE', 'USER.ROLE', 'READ'), -(66, 'ADMIN', 'RESOURCE', 'MCP', 'CREATE'), -(67, 'ADMIN', 'RESOURCE', 'MCP', 'READ'), -(68, 'ADMIN', 'RESOURCE', 'MCP', 'UPDATE'), -(69, 'ADMIN', 'RESOURCE', 'MCP', 'DELETE'), -(70, 'ADMIN', 'RESOURCE', 'MEM.SETTING', 'READ'), -(71, 'ADMIN', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(72, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'CREATE'), -(73, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'READ'), -(74, 'ADMIN', 'RESOURCE', 'MEM.AGENT', 'DELETE'), -(75, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(76, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(77, 'ADMIN', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(78, 'ADMIN', 'RESOURCE', 'MODEL', 'CREATE'), -(79, 'ADMIN', 'RESOURCE', 'MODEL', 'READ'), -(80, 'ADMIN', 'RESOURCE', 'MODEL', 'UPDATE'), -(81, 'ADMIN', 'RESOURCE', 'MODEL', 'DELETE'), -(82, 'ADMIN', 'RESOURCE', 'TENANT.INFO', 'READ'), -(83, 'ADMIN', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), -(84, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), -(85, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'READ'), -(86, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), -(87, 'ADMIN', 'RESOURCE', 'TENANT.INVITE', 'DELETE'), -(88, 'ADMIN', 'RESOURCE', 'GROUP', 'CREATE'), -(89, 'ADMIN', 'RESOURCE', 'GROUP', 'READ'), -(90, 'ADMIN', 'RESOURCE', 'GROUP', 'UPDATE'), -(91, 'ADMIN', 'RESOURCE', 'GROUP', 'DELETE'), -(92, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(93, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(94, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), -(95, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(96, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), -(97, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), -(98, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(99, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), -(100, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(101, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(102, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(103, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(104, 'DEV', 'RESOURCE', 'AGENT', 'CREATE'), -(105, 'DEV', 'RESOURCE', 'AGENT', 'READ'), -(106, 'DEV', 'RESOURCE', 'AGENT', 'UPDATE'), -(107, 'DEV', 'RESOURCE', 'AGENT', 'DELETE'), -(108, 'DEV', 'RESOURCE', 'KB', 'CREATE'), -(109, 'DEV', 'RESOURCE', 'KB', 'READ'), -(110, 'DEV', 'RESOURCE', 'KB', 'UPDATE'), -(111, 'DEV', 'RESOURCE', 'KB', 'DELETE'), -(112, 'DEV', 'RESOURCE', 'KB.GROUPS', 'READ'), -(113, 'DEV', 'RESOURCE', 'KB.GROUPS', 'UPDATE'), -(114, 'DEV', 'RESOURCE', 'KB.GROUPS', 'DELETE'), -(115, 'DEV', 'RESOURCE', 'USER.ROLE', 'READ'), -(116, 'DEV', 'RESOURCE', 'MCP', 'CREATE'), -(117, 'DEV', 'RESOURCE', 'MCP', 'READ'), -(118, 'DEV', 'RESOURCE', 'MCP', 'UPDATE'), -(119, 'DEV', 'RESOURCE', 'MCP', 'DELETE'), -(120, 'DEV', 'RESOURCE', 'MEM.SETTING', 'READ'), -(121, 'DEV', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(122, 'DEV', 'RESOURCE', 'MEM.AGENT', 'READ'), -(123, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(124, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(125, 'DEV', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(126, 'DEV', 'RESOURCE', 'MODEL', 'READ'), -(127, 'DEV', 'RESOURCE', 'TENANT.INFO', 'READ'), -(128, 'DEV', 'RESOURCE', 'GROUP', 'READ'), -(129, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(130, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(131, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(132, 'USER', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), -(133, 'USER', 'RESOURCE', 'AGENT', 'READ'), -(134, 'USER', 'RESOURCE', 'USER.ROLE', 'READ'), -(135, 'USER', 'RESOURCE', 'MEM.SETTING', 'READ'), -(136, 'USER', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(137, 'USER', 'RESOURCE', 'MEM.AGENT', 'READ'), -(138, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(139, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(140, 'USER', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(141, 'USER', 'RESOURCE', 'TENANT.INFO', 'READ'), -(142, 'USER', 'RESOURCE', 'GROUP', 'READ'), -(143, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(144, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), -(145, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/setup'), -(146, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), -(147, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), -(148, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), -(149, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), -(150, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-tools'), -(151, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/monitoring'), -(152, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), -(153, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/memory'), -(154, 'SPEED', 'RESOURCE', 'AGENT', 'CREATE'), -(155, 'SPEED', 'RESOURCE', 'AGENT', 'READ'), -(156, 'SPEED', 'RESOURCE', 'AGENT', 'UPDATE'), -(157, 'SPEED', 'RESOURCE', 'AGENT', 'DELETE'), -(158, 'SPEED', 'RESOURCE', 'KB', 'CREATE'), -(159, 'SPEED', 'RESOURCE', 'KB', 'READ'), -(160, 'SPEED', 'RESOURCE', 'KB', 'UPDATE'), -(161, 'SPEED', 'RESOURCE', 'KB', 'DELETE'), -(166, 'SPEED', 'RESOURCE', 'MCP', 'CREATE'), -(167, 'SPEED', 'RESOURCE', 'MCP', 'READ'), -(168, 'SPEED', 'RESOURCE', 'MCP', 'UPDATE'), -(169, 'SPEED', 'RESOURCE', 'MCP', 'DELETE'), -(170, 'SPEED', 'RESOURCE', 'MEM.SETTING', 'READ'), -(171, 'SPEED', 'RESOURCE', 'MEM.SETTING', 'UPDATE'), -(172, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'CREATE'), -(173, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'READ'), -(174, 'SPEED', 'RESOURCE', 'MEM.AGENT', 'DELETE'), -(175, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'CREATE'), -(176, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'READ'), -(177, 'SPEED', 'RESOURCE', 'MEM.PRIVATE', 'DELETE'), -(178, 'SPEED', 'RESOURCE', 'MODEL', 'CREATE'), -(179, 'SPEED', 'RESOURCE', 'MODEL', 'READ'), -(180, 'SPEED', 'RESOURCE', 'MODEL', 'UPDATE'), -(181, 'SPEED', 'RESOURCE', 'MODEL', 'DELETE'), -(182, 'SPEED', 'RESOURCE', 'TENANT.INFO', 'READ'), -(183, 'SPEED', 'RESOURCE', 'TENANT.INFO', 'UPDATE'), -(184, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'CREATE'), -(185, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'READ'), -(186, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'UPDATE'), -(187, 'SPEED', 'RESOURCE', 'TENANT.INVITE', 'DELETE') diff --git a/docker/sql/v1.8.1_0306_add_user_token_info.sql b/docker/sql/v1.8.1_0306_add_user_token_info.sql deleted file mode 100644 index 402cf4bab..000000000 --- a/docker/sql/v1.8.1_0306_add_user_token_info.sql +++ /dev/null @@ -1,76 +0,0 @@ --- Migration: Add user_token_info_t and user_token_usage_log_t tables --- Date: 2026-03-06 --- Description: Create user token (AK/SK) management tables with audit fields - --- Set search path to nexent schema -SET search_path TO nexent; - --- Create the user_token_info_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.user_token_info_t ( - token_id SERIAL4 PRIMARY KEY NOT NULL, - access_key VARCHAR(100) NOT NULL, - user_id VARCHAR(100) NOT NULL, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE "user_token_info_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.user_token_info_t IS 'User token (AK/SK) information table'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.user_token_info_t.token_id IS 'Token ID, unique primary key'; -COMMENT ON COLUMN nexent.user_token_info_t.access_key IS 'Access Key (AK)'; -COMMENT ON COLUMN nexent.user_token_info_t.user_id IS 'User ID who owns this token'; -COMMENT ON COLUMN nexent.user_token_info_t.create_time IS 'Creation time, audit field'; -COMMENT ON COLUMN nexent.user_token_info_t.update_time IS 'Update time, audit field'; -COMMENT ON COLUMN nexent.user_token_info_t.created_by IS 'Creator ID, audit field'; -COMMENT ON COLUMN nexent.user_token_info_t.updated_by IS 'Last updater ID, audit field'; -COMMENT ON COLUMN nexent.user_token_info_t.delete_flag IS 'Soft delete flag, Y means deleted'; - - --- Create the user_token_usage_log_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.user_token_usage_log_t ( - token_usage_id SERIAL4 PRIMARY KEY NOT NULL, - token_id INT4 NOT NULL, - call_function_name VARCHAR(100), - related_id INT4, - meta_data JSONB, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE "user_token_usage_log_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.user_token_usage_log_t IS 'User token usage log table'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.user_token_usage_log_t.token_usage_id IS 'Token usage log ID, unique primary key'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.token_id IS 'Foreign key to user_token_info_t.token_id'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.call_function_name IS 'API function name being called'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.related_id IS 'Related resource ID (e.g., conversation_id)'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.meta_data IS 'Additional metadata for this usage log entry, stored as JSON'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.create_time IS 'Creation time, audit field'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.update_time IS 'Update time, audit field'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.created_by IS 'Creator ID, audit field'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.updated_by IS 'Last updater ID, audit field'; -COMMENT ON COLUMN nexent.user_token_usage_log_t.delete_flag IS 'Soft delete flag, Y means deleted'; - --- Migration: Remove partner_mapping_id_t table for northbound conversation ID mapping --- Date: 2026-03-10 --- Description: Remove the external-internal conversation ID mapping table as northbound APIs now use internal conversation IDs directly --- Note: This table is no longer needed after refactoring northbound authentication logic - --- Drop the partner_mapping_id_t table if it exists -DROP TABLE IF EXISTS nexent.partner_mapping_id_t CASCADE; - --- Drop the associated sequence if it exists -DROP SEQUENCE IF EXISTS nexent.partner_mapping_id_t_id_seq; diff --git a/docker/sql/v2.0.0_0314_add_context_skill_t.sql b/docker/sql/v2.0.0_0314_add_context_skill_t.sql deleted file mode 100644 index 5fd23c97e..000000000 --- a/docker/sql/v2.0.0_0314_add_context_skill_t.sql +++ /dev/null @@ -1,105 +0,0 @@ --- Migration: Add ag_skill_info_t, ag_skill_tools_rel_t, and ag_skill_instance_t tables --- Date: 2026-03-14 --- Description: Create skill management tables with skill content, tags, and tool relationships - -SET search_path TO nexent; - --- Create the ag_skill_info_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.ag_skill_info_t ( - skill_id SERIAL4 PRIMARY KEY NOT NULL, - skill_name VARCHAR(100) NOT NULL, - skill_description VARCHAR(1000), - skill_tags JSON, - skill_content TEXT, - params JSON, - source VARCHAR(30) DEFAULT 'official', - created_by VARCHAR(100), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - updated_by VARCHAR(100), - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE "ag_skill_info_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.ag_skill_info_t IS 'Skill information table for managing custom skills'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_skill_info_t.skill_id IS 'Skill ID, unique primary key'; -COMMENT ON COLUMN nexent.ag_skill_info_t.skill_name IS 'Skill name, globally unique'; -COMMENT ON COLUMN nexent.ag_skill_info_t.skill_description IS 'Skill description text'; -COMMENT ON COLUMN nexent.ag_skill_info_t.skill_tags IS 'Skill tags stored as JSON array'; -COMMENT ON COLUMN nexent.ag_skill_info_t.skill_content IS 'Skill content or prompt text'; -COMMENT ON COLUMN nexent.ag_skill_info_t.params IS 'Skill configuration parameters stored as JSON object'; -COMMENT ON COLUMN nexent.ag_skill_info_t.source IS 'Skill source: official, custom, or partner'; -COMMENT ON COLUMN nexent.ag_skill_info_t.created_by IS 'Creator ID'; -COMMENT ON COLUMN nexent.ag_skill_info_t.create_time IS 'Creation timestamp'; -COMMENT ON COLUMN nexent.ag_skill_info_t.updated_by IS 'Last updater ID'; -COMMENT ON COLUMN nexent.ag_skill_info_t.update_time IS 'Last update timestamp'; -COMMENT ON COLUMN nexent.ag_skill_info_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create the ag_skill_tools_rel_t table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.ag_skill_tools_rel_t ( - rel_id SERIAL4 PRIMARY KEY NOT NULL, - skill_id INTEGER, - tool_id INTEGER, - created_by VARCHAR(100), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - updated_by VARCHAR(100), - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE "ag_skill_tools_rel_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.ag_skill_tools_rel_t IS 'Skill-tool relationship table for many-to-many mapping'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.rel_id IS 'Relationship ID, unique primary key'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.skill_id IS 'Foreign key to ag_skill_info_t.skill_id'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.tool_id IS 'Tool ID from ag_tool_info_t'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.created_by IS 'Creator ID'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.create_time IS 'Creation timestamp'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.updated_by IS 'Last updater ID'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.update_time IS 'Last update timestamp'; -COMMENT ON COLUMN nexent.ag_skill_tools_rel_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create the ag_skill_instance_t table in the nexent schema --- Stores skill instance configuration per agent version --- Note: skill_description and skill_content fields removed, now retrieved from ag_skill_info_t -CREATE TABLE IF NOT EXISTS nexent.ag_skill_instance_t ( - skill_instance_id SERIAL4 NOT NULL, - skill_id INTEGER NOT NULL, - agent_id INTEGER NOT NULL, - user_id VARCHAR(100), - tenant_id VARCHAR(100), - enabled BOOLEAN DEFAULT TRUE, - version_no INTEGER DEFAULT 0 NOT NULL, - created_by VARCHAR(100), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - updated_by VARCHAR(100), - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - delete_flag VARCHAR(1) DEFAULT 'N', - CONSTRAINT ag_skill_instance_t_pkey PRIMARY KEY (skill_instance_id, version_no) -); - -ALTER TABLE "ag_skill_instance_t" OWNER TO "root"; - --- Add comment to the table -COMMENT ON TABLE nexent.ag_skill_instance_t IS 'Skill instance configuration table - stores per-agent skill settings'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_skill_instance_t.skill_instance_id IS 'Skill instance ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.skill_id IS 'Foreign key to ag_skill_info_t.skill_id'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.agent_id IS 'Agent ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.user_id IS 'User ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.tenant_id IS 'Tenant ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.enabled IS 'Whether this skill is enabled for the agent'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.version_no IS 'Version number. 0 = draft/editing state, >=1 = published snapshot'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.created_by IS 'Creator ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.create_time IS 'Creation timestamp'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.updated_by IS 'Last updater ID'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.update_time IS 'Last update timestamp'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; diff --git a/docker/sql/v2.0.1_0331_add_outer_api_tool_t.sql b/docker/sql/v2.0.1_0331_add_outer_api_tool_t.sql deleted file mode 100644 index b6e055775..000000000 --- a/docker/sql/v2.0.1_0331_add_outer_api_tool_t.sql +++ /dev/null @@ -1,70 +0,0 @@ --- v2.0.1_0331_add_outer_api_tool_t.sql --- Create table for outer API tools (OpenAPI to MCP conversion) - --- Create the ag_outer_api_tools table in the nexent schema -CREATE TABLE IF NOT EXISTS nexent.ag_outer_api_tools ( - id BIGSERIAL PRIMARY KEY, - name VARCHAR(100) NOT NULL, - description TEXT, - method VARCHAR(10), - url TEXT NOT NULL, - headers_template JSONB DEFAULT '{}', - query_template JSONB DEFAULT '{}', - body_template JSONB DEFAULT '{}', - input_schema JSONB DEFAULT '{}', - tenant_id VARCHAR(100), - is_available BOOLEAN DEFAULT TRUE, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.ag_outer_api_tools OWNER TO "root"; - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_ag_outer_api_tools_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Create a trigger to call the function before each update -CREATE TRIGGER update_ag_outer_api_tools_update_time_trigger -BEFORE UPDATE ON nexent.ag_outer_api_tools -FOR EACH ROW -EXECUTE FUNCTION update_ag_outer_api_tools_update_time(); - --- Add comment to the table -COMMENT ON TABLE nexent.ag_outer_api_tools IS 'Outer API tools table - stores converted OpenAPI tools as MCP tools'; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_outer_api_tools.id IS 'Tool ID, unique primary key'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.name IS 'Tool name (unique identifier)'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.description IS 'Tool description'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.method IS 'HTTP method: GET/POST/PUT/DELETE/PATCH'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.url IS 'API endpoint URL (full path with base URL)'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.headers_template IS 'Headers template as JSONB'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.query_template IS 'Query parameters template as JSONB'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.body_template IS 'Request body template as JSONB'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.input_schema IS 'MCP input schema as JSONB'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.tenant_id IS 'Tenant ID for multi-tenancy'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.is_available IS 'Whether the tool is available'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.created_by IS 'Creator'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.updated_by IS 'Updater'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create index for tenant_id queries -CREATE INDEX IF NOT EXISTS idx_ag_outer_api_tools_tenant_id -ON nexent.ag_outer_api_tools (tenant_id) -WHERE delete_flag = 'N'; - --- Create index for name queries -CREATE INDEX IF NOT EXISTS idx_ag_outer_api_tools_name -ON nexent.ag_outer_api_tools (name) -WHERE delete_flag = 'N'; diff --git a/docker/sql/v2.0.2_0410_add_columns_outer_api_tools.sql b/docker/sql/v2.0.2_0410_add_columns_outer_api_tools.sql deleted file mode 100644 index fe527cf16..000000000 --- a/docker/sql/v2.0.2_0410_add_columns_outer_api_tools.sql +++ /dev/null @@ -1,19 +0,0 @@ --- v2.0.2_0410_add_columns_outer_api_tools.sql --- Add MCP service-level columns to ag_outer_api_tools table --- These columns enable grouping tools from the same OpenAPI spec under a single MCP service - --- Add columns for MCP service information -ALTER TABLE nexent.ag_outer_api_tools - ADD COLUMN IF NOT EXISTS mcp_service_name VARCHAR(100), - ADD COLUMN IF NOT EXISTS openapi_json JSONB, - ADD COLUMN IF NOT EXISTS server_url VARCHAR(500); - --- Add comments to the new columns -COMMENT ON COLUMN nexent.ag_outer_api_tools.mcp_service_name IS 'MCP service name for grouping tools from same OpenAPI spec'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.openapi_json IS 'Complete OpenAPI JSON specification'; -COMMENT ON COLUMN nexent.ag_outer_api_tools.server_url IS 'Base URL of the REST API server'; - --- Create index for mcp_service_name queries -CREATE INDEX IF NOT EXISTS idx_ag_outer_api_tools_mcp_service_name -ON nexent.ag_outer_api_tools (mcp_service_name) -WHERE delete_flag = 'N' AND mcp_service_name IS NOT NULL; \ No newline at end of file diff --git a/docker/sql/v2.0.2_0414_migrate_outer_api_tools_to_services.sql b/docker/sql/v2.0.2_0414_migrate_outer_api_tools_to_services.sql deleted file mode 100644 index 130cffdde..000000000 --- a/docker/sql/v2.0.2_0414_migrate_outer_api_tools_to_services.sql +++ /dev/null @@ -1,65 +0,0 @@ --- Migration: Convert ag_outer_api_tools (tool-level) to ag_outer_api_services (service-level) --- Date: 2026-04-09 --- Description: Each OpenAPI service now stores one record instead of one record per tool. --- Only service-level fields (mcp_service_name, openapi_json, server_url, etc.) are kept. - --- Step 1: Create new table for services -CREATE TABLE IF NOT EXISTS nexent.ag_outer_api_services ( - id BIGSERIAL PRIMARY KEY, - mcp_service_name VARCHAR(100) NOT NULL, - description TEXT, - openapi_json JSONB, - server_url VARCHAR(500), - headers_template JSONB, - tenant_id VARCHAR(100) NOT NULL, - is_available BOOLEAN DEFAULT TRUE, - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Step 2: Migrate data - one record per service --- Use DISTINCT ON to get one record per (tenant_id, mcp_service_name) --- Order by update_time DESC to keep the most recently updated record -INSERT INTO nexent.ag_outer_api_services ( - mcp_service_name, - description, - openapi_json, - server_url, - headers_template, - tenant_id, - is_available, - create_time, - update_time, - created_by, - updated_by, - delete_flag -) -SELECT DISTINCT ON (t.tenant_id, t.mcp_service_name) - t.mcp_service_name, - t.description, - t.openapi_json, - t.server_url, - t.headers_template, - t.tenant_id, - COALESCE(t.is_available, TRUE) as is_available, - t.create_time, - t.update_time, - t.created_by, - t.updated_by, - t.delete_flag -FROM nexent.ag_outer_api_tools t -WHERE t.delete_flag != 'Y' -ORDER BY t.tenant_id, t.mcp_service_name, t.update_time DESC -ON CONFLICT DO NOTHING; - --- Step 3: Verify migration -SELECT 'Migrated services count: ' || COUNT(*) FROM nexent.ag_outer_api_services; - --- Step 4: Drop old table after successful migration -DROP TABLE IF EXISTS nexent.ag_outer_api_tools; - --- Step 5: Drop the old sequence (no longer needed) -DROP SEQUENCE IF EXISTS nexent.ag_outer_api_tools_id_seq; diff --git a/docker/sql/v2.0.2_0420_add_fk_to_ag_a2a_message_t.sql b/docker/sql/v2.0.2_0420_add_fk_to_ag_a2a_message_t.sql deleted file mode 100644 index 6391ec349..000000000 --- a/docker/sql/v2.0.2_0420_add_fk_to_ag_a2a_message_t.sql +++ /dev/null @@ -1,14 +0,0 @@ --- ============================================================================= --- Add Foreign Key Constraint to ag_a2a_message_t --- ============================================================================= --- Version: v2.0.2 --- Date: 2026-04-20 --- Description: Add foreign key constraint on task_id referencing ag_a2a_task_t(id) --- Target Table: nexent.ag_a2a_message_t --- ============================================================================= - --- Add foreign key constraint: task_id references ag_a2a_task_t(id) with CASCADE delete -ALTER TABLE nexent.ag_a2a_message_t - ADD CONSTRAINT ag_a2a_message_t_task_id_fk - FOREIGN KEY (task_id) - REFERENCES nexent.ag_a2a_task_t(id) ON DELETE CASCADE; diff --git a/docker/sql/v2.0.2_0425_add_is_a2a_to_ag_tenant_agent_version_t.sql b/docker/sql/v2.0.2_0425_add_is_a2a_to_ag_tenant_agent_version_t.sql deleted file mode 100644 index 3eb6ac5e9..000000000 --- a/docker/sql/v2.0.2_0425_add_is_a2a_to_ag_tenant_agent_version_t.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Add is_a2a column to ag_tenant_agent_version_t for tracking A2A Server agent publish status --- This field indicates whether this version was published as an A2A Server agent - -ALTER TABLE nexent.ag_tenant_agent_version_t -ADD COLUMN IF NOT EXISTS is_a2a BOOLEAN DEFAULT FALSE; - -COMMENT ON COLUMN nexent.ag_tenant_agent_version_t.is_a2a IS 'Whether this version is published as an A2A Server agent'; diff --git a/docker/sql/v2.0.3_0423_create_model_monitoring_record_t.sql b/docker/sql/v2.0.3_0423_create_model_monitoring_record_t.sql deleted file mode 100644 index 438ca4863..000000000 --- a/docker/sql/v2.0.3_0423_create_model_monitoring_record_t.sql +++ /dev/null @@ -1,42 +0,0 @@ --- Model Monitoring Record Table --- Stores per-request LLM performance metrics for the monitoring feature. --- Run this script against the 'nexent' schema in PostgreSQL. - -CREATE TABLE IF NOT EXISTS nexent.model_monitoring_record_t ( - monitoring_id SERIAL PRIMARY KEY, - model_id INT4, - model_name VARCHAR(100) NOT NULL, - model_type VARCHAR(20) DEFAULT 'llm', - agent_id INT4, - agent_name VARCHAR(100), - conversation_id INT4, - tenant_id VARCHAR(100) NOT NULL, - user_id VARCHAR(100), - display_name VARCHAR(100), - request_duration_ms INT4, - ttft_ms INT4, - input_tokens INT4, - output_tokens INT4, - total_tokens INT4, - generation_rate FLOAT, - is_streaming BOOLEAN DEFAULT FALSE, - is_success BOOLEAN DEFAULT TRUE, - is_error BOOLEAN DEFAULT FALSE, - error_type VARCHAR(50), - error_message TEXT, - retry_count INT4 DEFAULT 0, - operation VARCHAR(50), - create_time TIMESTAMP DEFAULT NOW(), - delete_flag VARCHAR(1) DEFAULT 'N' -); - --- Single-column indexes for common query patterns -CREATE INDEX IF NOT EXISTS ix_monitoring_model_id ON nexent.model_monitoring_record_t (model_id); -CREATE INDEX IF NOT EXISTS ix_monitoring_tenant_id ON nexent.model_monitoring_record_t (tenant_id); -CREATE INDEX IF NOT EXISTS ix_monitoring_agent_id ON nexent.model_monitoring_record_t (agent_id); -CREATE INDEX IF NOT EXISTS ix_monitoring_create_time ON nexent.model_monitoring_record_t (create_time); -CREATE INDEX IF NOT EXISTS ix_monitoring_is_error ON nexent.model_monitoring_record_t (is_error); -CREATE INDEX IF NOT EXISTS ix_monitoring_model_type ON nexent.model_monitoring_record_t (model_type); - --- Composite index for time-range queries per model -CREATE INDEX IF NOT EXISTS ix_monitoring_model_time ON nexent.model_monitoring_record_t (model_id, create_time); diff --git a/docker/sql/v2.0.3_0430_add_user_oauth_account_t.sql b/docker/sql/v2.0.3_0430_add_user_oauth_account_t.sql deleted file mode 100644 index faa9adab2..000000000 --- a/docker/sql/v2.0.3_0430_add_user_oauth_account_t.sql +++ /dev/null @@ -1,52 +0,0 @@ --- Create user OAuth account table for third-party login (GitHub, WeChat, etc.) -CREATE TABLE IF NOT EXISTS nexent.user_oauth_account_t ( - oauth_account_id SERIAL PRIMARY KEY, - user_id VARCHAR(100) NOT NULL, - provider VARCHAR(30) NOT NULL, - provider_user_id VARCHAR(200) NOT NULL, - provider_email VARCHAR(255), - provider_username VARCHAR(200), - tenant_id VARCHAR(100), - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag CHAR(1) DEFAULT 'N', - CONSTRAINT uq_oauth_provider_user UNIQUE (provider, provider_user_id) -); - -ALTER TABLE nexent.user_oauth_account_t OWNER TO "root"; - --- Create a function to update the update_time column -CREATE OR REPLACE FUNCTION update_user_oauth_account_t_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - --- Create a trigger to call the function before each update -CREATE TRIGGER update_user_oauth_account_t_update_time_trigger -BEFORE UPDATE ON nexent.user_oauth_account_t -FOR EACH ROW -EXECUTE FUNCTION update_user_oauth_account_t_update_time(); - --- Add comments -COMMENT ON TABLE nexent.user_oauth_account_t IS 'User OAuth account table - third-party login bindings'; -COMMENT ON COLUMN nexent.user_oauth_account_t.oauth_account_id IS 'OAuth account ID, primary key'; -COMMENT ON COLUMN nexent.user_oauth_account_t.user_id IS 'Nexent user ID (Supabase UUID)'; -COMMENT ON COLUMN nexent.user_oauth_account_t.provider IS 'OAuth provider name: github, wechat, gde, link_app'; -COMMENT ON COLUMN nexent.user_oauth_account_t.provider_user_id IS 'User ID from the OAuth provider'; -COMMENT ON COLUMN nexent.user_oauth_account_t.provider_email IS 'Email from the OAuth provider'; -COMMENT ON COLUMN nexent.user_oauth_account_t.provider_username IS 'Display name from the OAuth provider'; -COMMENT ON COLUMN nexent.user_oauth_account_t.tenant_id IS 'Tenant ID at time of linking'; -COMMENT ON COLUMN nexent.user_oauth_account_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.user_oauth_account_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.user_oauth_account_t.created_by IS 'Creator'; -COMMENT ON COLUMN nexent.user_oauth_account_t.updated_by IS 'Updater'; -COMMENT ON COLUMN nexent.user_oauth_account_t.delete_flag IS 'Whether it is deleted. Optional values: Y/N'; - --- Create index for user_id queries -CREATE INDEX IF NOT EXISTS idx_user_oauth_account_t_user_id -ON nexent.user_oauth_account_t (user_id); diff --git a/docker/sql/v2.0.4_0427_add_enable_context_manager_to_ag_tenant_agent_t.sql b/docker/sql/v2.0.4_0427_add_enable_context_manager_to_ag_tenant_agent_t.sql deleted file mode 100644 index b89a19e04..000000000 --- a/docker/sql/v2.0.4_0427_add_enable_context_manager_to_ag_tenant_agent_t.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Migration: Add enable_context_manager column to ag_tenant_agent_t table --- Date: 2025-04-27 --- Description: Add enable_context_manager field to control context management (compression) per agent - --- Add enable_context_manager column to ag_tenant_agent_t table -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS enable_context_manager BOOLEAN DEFAULT FALSE; - --- Add comment to the column -COMMENT ON COLUMN nexent.ag_tenant_agent_t.enable_context_manager IS 'Whether to enable context management (compression) for this agent'; \ No newline at end of file diff --git a/docker/sql/v2.0.4_0506_add_base_url_in_external_agent.sql b/docker/sql/v2.0.4_0506_add_base_url_in_external_agent.sql deleted file mode 100644 index e4723bc96..000000000 --- a/docker/sql/v2.0.4_0506_add_base_url_in_external_agent.sql +++ /dev/null @@ -1,13 +0,0 @@ -ALTER TABLE nexent.ag_a2a_external_agent_t -ADD COLUMN IF NOT EXISTS base_url VARCHAR(512); - -COMMENT ON COLUMN nexent.ag_a2a_external_agent_t.base_url IS 'Base URL for health checks (service root address)'; - -ALTER TABLE nexent.ag_a2a_message_t - DROP CONSTRAINT IF EXISTS ag_a2a_message_t_task_id_fk; - -ALTER TABLE nexent.ag_a2a_external_agent_relation_t - DROP CONSTRAINT IF EXISTS fk_external_agent; - -ALTER TABLE nexent.ag_a2a_artifact_t - DROP CONSTRAINT IF EXISTS fk_artifact_task; \ No newline at end of file diff --git a/docker/sql/v2.0.5_0511_add_auto_summary_fields_to_knowledge_record_t.sql b/docker/sql/v2.0.5_0511_add_auto_summary_fields_to_knowledge_record_t.sql deleted file mode 100644 index 491f6b27b..000000000 --- a/docker/sql/v2.0.5_0511_add_auto_summary_fields_to_knowledge_record_t.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Migration: Add auto-summary fields to knowledge_record_t table --- Date: 2026-05-11 --- Description: Add summary_frequency, last_summary_time, and last_doc_update_time fields for auto-summary feature --- This SQL consolidates fields added in multiple commits for clean upgrade path - --- Add summary_frequency column (auto-summary frequency configuration) -ALTER TABLE nexent.knowledge_record_t -ADD COLUMN IF NOT EXISTS summary_frequency VARCHAR(10); - --- Add last_summary_time column (timestamp of last summary generation) -ALTER TABLE nexent.knowledge_record_t -ADD COLUMN IF NOT EXISTS last_summary_time TIMESTAMP; - --- Add last_doc_update_time column (timestamp of last document add/delete operation) -ALTER TABLE nexent.knowledge_record_t -ADD COLUMN IF NOT EXISTS last_doc_update_time TIMESTAMP; - --- Add comments to the columns -COMMENT ON COLUMN nexent.knowledge_record_t.summary_frequency IS 'Auto-summary frequency: 1h, 3h, 6h, 1d, 1w, or NULL (disabled)'; -COMMENT ON COLUMN nexent.knowledge_record_t.last_summary_time IS 'Timestamp of last summary generation'; -COMMENT ON COLUMN nexent.knowledge_record_t.last_doc_update_time IS 'Timestamp of last document add/delete operation, used for auto-summary optimization to skip unnecessary summary regeneration'; \ No newline at end of file diff --git a/docker/sql/v2.1.1_0508_add_embedding_model_id_to_knowledge_record_t.sql b/docker/sql/v2.1.1_0508_add_embedding_model_id_to_knowledge_record_t.sql deleted file mode 100644 index 0305a2590..000000000 --- a/docker/sql/v2.1.1_0508_add_embedding_model_id_to_knowledge_record_t.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Add embedding_model_id column to knowledge_record_t table --- This field stores the ID of the embedding model used by the knowledge base - --- Add embedding_model_id column -ALTER TABLE "knowledge_record_t" -ADD COLUMN IF NOT EXISTS "embedding_model_id" INTEGER; - --- Add column comment -COMMENT ON COLUMN "knowledge_record_t"."embedding_model_id" IS 'Embedding model ID, foreign key reference to model_record_t.model_id'; diff --git a/docker/sql/v2.1.1_0509_add_model_appid_token_to_model_record_t.sql b/docker/sql/v2.1.1_0509_add_model_appid_token_to_model_record_t.sql deleted file mode 100644 index 521fa38a4..000000000 --- a/docker/sql/v2.1.1_0509_add_model_appid_token_to_model_record_t.sql +++ /dev/null @@ -1,9 +0,0 @@ -ALTER TABLE nexent.model_record_t -ADD COLUMN IF NOT EXISTS model_appid VARCHAR(100) DEFAULT ''; - - -ALTER TABLE nexent.model_record_t -ADD COLUMN IF NOT EXISTS access_token VARCHAR(100) DEFAULT ''; - -COMMENT ON COLUMN nexent.model_record_t.model_appid IS 'Application ID for model authentication.'; -COMMENT ON COLUMN nexent.model_record_t.access_token IS 'Access token for model authentication.'; diff --git a/docker/sql/v2.2.0_0514_skill_config_schema.sql b/docker/sql/v2.2.0_0514_skill_config_schema.sql deleted file mode 100644 index 12e549175..000000000 --- a/docker/sql/v2.2.0_0514_skill_config_schema.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Rename params -> config_values, add config_schemas to ag_skill_info_t --- Add tenant_id column for multi-tenancy support -ALTER TABLE nexent.ag_skill_info_t ADD COLUMN IF NOT EXISTS tenant_id VARCHAR(100); - --- Add config_values and config_schemas to ag_skill_info_t -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = 'nexent' - AND table_name = 'ag_skill_info_t' - AND column_name = 'params' - ) THEN - ALTER TABLE nexent.ag_skill_info_t RENAME COLUMN params TO config_values; - END IF; -END $$; -ALTER TABLE nexent.ag_skill_info_t ADD COLUMN IF NOT EXISTS config_schemas JSON; - --- Comments for ag_skill_info_t columns -COMMENT ON COLUMN nexent.ag_skill_info_t.tenant_id IS 'Tenant ID for multi-tenancy. NULL for pre-existing skills.'; -COMMENT ON COLUMN nexent.ag_skill_info_t.config_values IS 'Runtime parameter values from config/config.yaml'; -COMMENT ON COLUMN nexent.ag_skill_info_t.config_schemas IS 'Parameter metadata list from config/schema.yaml'; - --- Add config_values and config_schemas to ag_skill_instance_t -ALTER TABLE nexent.ag_skill_instance_t ADD COLUMN IF NOT EXISTS config_values JSON; -ALTER TABLE nexent.ag_skill_instance_t ADD COLUMN IF NOT EXISTS config_schemas JSON; - --- Comments for ag_skill_instance_t columns -COMMENT ON COLUMN nexent.ag_skill_instance_t.config_values IS 'Per-agent runtime parameter values from config/config.yaml'; -COMMENT ON COLUMN nexent.ag_skill_instance_t.config_schemas IS 'Per-agent parameter schema overrides from config/schema.yaml'; diff --git a/docker/sql/v2.2.0_0520_add_concurrency_and_timeout_to_model_record_t.sql b/docker/sql/v2.2.0_0520_add_concurrency_and_timeout_to_model_record_t.sql deleted file mode 100644 index 59632f8ed..000000000 --- a/docker/sql/v2.2.0_0520_add_concurrency_and_timeout_to_model_record_t.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Add concurrency_limit column to model_record_t table -ALTER TABLE nexent.model_record_t -ADD COLUMN IF NOT EXISTS concurrency_limit INTEGER DEFAULT NULL; - --- Add comment to the column -COMMENT ON COLUMN nexent.model_record_t.concurrency_limit IS 'Maximum concurrent requests for this model. Default is NULL (unlimited).'; - --- Add timeout_seconds column to model_record_t table -ALTER TABLE nexent.model_record_t -ADD COLUMN IF NOT EXISTS timeout_seconds INTEGER DEFAULT 120; - --- Add comment to the column -COMMENT ON COLUMN nexent.model_record_t.timeout_seconds IS 'Request timeout in seconds for this model. Default is 120 seconds.'; diff --git a/docker/sql/v2.2.0_0521_add_mcp_community_record_t.sql b/docker/sql/v2.2.0_0521_add_mcp_community_record_t.sql deleted file mode 100644 index 83f9d9a56..000000000 --- a/docker/sql/v2.2.0_0521_add_mcp_community_record_t.sql +++ /dev/null @@ -1,83 +0,0 @@ --- Migration: Add mcp_community_record_t table --- Date: 2026-03-26 --- Description: Community MCP market table aligned with public-shareable fields from mcp_record_t. - -SET search_path TO nexent; - -BEGIN; - -CREATE TABLE IF NOT EXISTS nexent.mcp_community_record_t ( - community_id SERIAL PRIMARY KEY NOT NULL, - tenant_id VARCHAR(100), - user_id VARCHAR(100), - mcp_name VARCHAR(100) NOT NULL, - mcp_server VARCHAR(500) NOT NULL, - source VARCHAR(30) DEFAULT 'community', - version VARCHAR(50), - registry_json JSONB, - transport_type VARCHAR(30), - config_json JSON, - tags TEXT[], - description TEXT, - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -ALTER TABLE nexent.mcp_community_record_t OWNER TO root; - -COMMENT ON TABLE nexent.mcp_community_record_t IS 'Community MCP market records, publishable from tenant MCP services'; -COMMENT ON COLUMN nexent.mcp_community_record_t.community_id IS 'Community record ID, unique primary key'; -COMMENT ON COLUMN nexent.mcp_community_record_t.tenant_id IS 'Publisher tenant ID'; -COMMENT ON COLUMN nexent.mcp_community_record_t.user_id IS 'Publisher user ID'; -COMMENT ON COLUMN nexent.mcp_community_record_t.mcp_name IS 'MCP name'; -COMMENT ON COLUMN nexent.mcp_community_record_t.mcp_server IS 'MCP server URL'; -COMMENT ON COLUMN nexent.mcp_community_record_t.source IS 'Source type, fixed to community for this table'; -COMMENT ON COLUMN nexent.mcp_community_record_t.version IS 'MCP version'; -COMMENT ON COLUMN nexent.mcp_community_record_t.registry_json IS 'Full MCP server metadata JSON for discovery and quick import'; -COMMENT ON COLUMN nexent.mcp_community_record_t.transport_type IS 'Transport type: url/container'; -COMMENT ON COLUMN nexent.mcp_community_record_t.config_json IS 'Public-shareable MCP configuration JSON'; -COMMENT ON COLUMN nexent.mcp_community_record_t.tags IS 'Tags'; -COMMENT ON COLUMN nexent.mcp_community_record_t.description IS 'Description'; -COMMENT ON COLUMN nexent.mcp_community_record_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.mcp_community_record_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.mcp_community_record_t.created_by IS 'Creator ID'; -COMMENT ON COLUMN nexent.mcp_community_record_t.updated_by IS 'Updater ID'; -COMMENT ON COLUMN nexent.mcp_community_record_t.delete_flag IS 'Soft delete flag: Y/N'; - -CREATE INDEX IF NOT EXISTS idx_mcp_community_tenant_delete - ON nexent.mcp_community_record_t (tenant_id, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_community_name_delete - ON nexent.mcp_community_record_t (mcp_name, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_community_transport_delete - ON nexent.mcp_community_record_t (transport_type, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_community_user_delete - ON nexent.mcp_community_record_t (user_id, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_community_tags_gin - ON nexent.mcp_community_record_t USING GIN (tags); - -CREATE OR REPLACE FUNCTION update_mcp_community_record_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -COMMENT ON FUNCTION update_mcp_community_record_update_time() IS 'Auto-update update_time for mcp_community_record_t'; - -DROP TRIGGER IF EXISTS update_mcp_community_record_update_time_trigger ON nexent.mcp_community_record_t; -CREATE TRIGGER update_mcp_community_record_update_time_trigger -BEFORE UPDATE ON nexent.mcp_community_record_t -FOR EACH ROW -EXECUTE FUNCTION update_mcp_community_record_update_time(); - -COMMENT ON TRIGGER update_mcp_community_record_update_time_trigger ON nexent.mcp_community_record_t IS 'Trigger to maintain update_time'; - -COMMIT; diff --git a/docker/sql/v2.2.0_0521_expand_mcp_record_t.sql b/docker/sql/v2.2.0_0521_expand_mcp_record_t.sql deleted file mode 100644 index 6c92a392e..000000000 --- a/docker/sql/v2.2.0_0521_expand_mcp_record_t.sql +++ /dev/null @@ -1,41 +0,0 @@ --- Migration: Extend mcp_record_t for MCP tools (direct schema) --- Date: 2026-03-18 --- Description: One-step schema extension for mcp_record_t. No table merge, no data migration. - -SET search_path TO nexent; - -BEGIN; - --- 1) Extend mcp_record_t with final column names (idempotent) -ALTER TABLE IF EXISTS nexent.mcp_record_t - ADD COLUMN IF NOT EXISTS source VARCHAR(30), - ADD COLUMN IF NOT EXISTS registry_json JSONB, - ADD COLUMN IF NOT EXISTS config_json JSON, - ADD COLUMN IF NOT EXISTS enabled BOOLEAN DEFAULT TRUE, - ADD COLUMN IF NOT EXISTS tags TEXT[], - ADD COLUMN IF NOT EXISTS description TEXT, - ADD COLUMN IF NOT EXISTS container_port INTEGER; - --- 2) Add comments for new columns -COMMENT ON COLUMN nexent.mcp_record_t.source IS 'Source type: local/mcp_registry/community'; -COMMENT ON COLUMN nexent.mcp_record_t.registry_json IS 'Full MCP registry server.json snapshot'; -COMMENT ON COLUMN nexent.mcp_record_t.config_json IS 'MCP config data'; -COMMENT ON COLUMN nexent.mcp_record_t.enabled IS 'Enabled'; -COMMENT ON COLUMN nexent.mcp_record_t.tags IS 'Tags'; -COMMENT ON COLUMN nexent.mcp_record_t.description IS 'Description'; -COMMENT ON COLUMN nexent.mcp_record_t.container_port IS 'Host port bound for containerized MCP service'; - --- 3) Add indexes for common management queries -CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tenant_delete - ON nexent.mcp_record_t (tenant_id, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tenant_name - ON nexent.mcp_record_t (tenant_id, mcp_name, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tenant_server - ON nexent.mcp_record_t (tenant_id, mcp_server, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_mcp_record_t_tags_gin - ON nexent.mcp_record_t USING GIN (tags); - -COMMIT; diff --git a/docker/sql/v2.2.0_0526_add_cas_session_t.sql b/docker/sql/v2.2.0_0526_add_cas_session_t.sql deleted file mode 100644 index 3f1aab4fa..000000000 --- a/docker/sql/v2.2.0_0526_add_cas_session_t.sql +++ /dev/null @@ -1,27 +0,0 @@ -CREATE TABLE IF NOT EXISTS nexent.user_cas_session_t ( - cas_session_id SERIAL PRIMARY KEY, - session_id VARCHAR(100) NOT NULL UNIQUE, - user_id VARCHAR(100) NOT NULL, - cas_user_id VARCHAR(200) NOT NULL, - cas_session_index VARCHAR(500), - status VARCHAR(30) NOT NULL DEFAULT 'active', - expires_at TIMESTAMP NOT NULL, - revoked_at TIMESTAMP, - create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N' -); - -CREATE INDEX IF NOT EXISTS ix_user_cas_session_session_id - ON nexent.user_cas_session_t (session_id); -CREATE INDEX IF NOT EXISTS ix_user_cas_session_user_id - ON nexent.user_cas_session_t (user_id); -CREATE INDEX IF NOT EXISTS ix_user_cas_session_cas_user_id - ON nexent.user_cas_session_t (cas_user_id); - -COMMENT ON TABLE nexent.user_cas_session_t IS 'Server-side session records for CAS SSO login and logout synchronization'; -COMMENT ON COLUMN nexent.user_cas_session_t.session_id IS 'JWT sid claim for revocation checks'; -COMMENT ON COLUMN nexent.user_cas_session_t.cas_user_id IS 'User identifier returned by CAS'; -COMMENT ON COLUMN nexent.user_cas_session_t.cas_session_index IS 'CAS SessionIndex or service ticket'; diff --git a/docker/sql/v2.2.0_0527_add_custom_headers_to_mcp_record_t.sql b/docker/sql/v2.2.0_0527_add_custom_headers_to_mcp_record_t.sql deleted file mode 100644 index 00933c523..000000000 --- a/docker/sql/v2.2.0_0527_add_custom_headers_to_mcp_record_t.sql +++ /dev/null @@ -1,26 +0,0 @@ --- Migration: Add custom_headers column to mcp_record_t --- Date: 2026-05-26 --- Description: Add custom_headers field to store custom HTTP headers for MCP server requests - -SET search_path TO nexent; - -BEGIN; - --- Add custom_headers column if it doesn't exist -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = 'nexent' - AND table_name = 'mcp_record_t' - AND column_name = 'custom_headers' - ) THEN - ALTER TABLE nexent.mcp_record_t - ADD COLUMN custom_headers JSON DEFAULT NULL; - END IF; -END $$; - --- Add comment to the column -COMMENT ON COLUMN nexent.mcp_record_t.custom_headers IS 'Custom HTTP headers as JSON object for MCP server requests'; - -COMMIT; diff --git a/docker/sql/v2.2.0_0529_add_asset_owner_role_permissions.sql b/docker/sql/v2.2.0_0529_add_asset_owner_role_permissions.sql deleted file mode 100644 index 8f21b110b..000000000 --- a/docker/sql/v2.2.0_0529_add_asset_owner_role_permissions.sql +++ /dev/null @@ -1,53 +0,0 @@ --- Migration: ASSET_OWNER role permissions and invitation type comment --- Date: 2026-05-29 --- Description: Add ASSET_OWNER role permissions, SU asset-owner invite permissions, --- update invitation code_type comment, and ensure ag_skill_info_t.tenant_id exists --- Source: commit 15cece97692db2372a978cbdf21b5d5316e79f30 (init.sql) - -SET search_path TO nexent; - -BEGIN; - -COMMENT ON COLUMN nexent.tenant_invitation_code_t.code_type IS - 'Invitation code type: ADMIN_INVITE, DEV_INVITE, USER_INVITE, ASSET_OWNER_INVITE'; - -INSERT INTO nexent.role_permission_t - (role_permission_id, user_role, permission_category, permission_type, permission_subtype) -VALUES - (188, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'CREATE'), - (189, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'READ'), - (190, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'UPDATE'), - (191, 'SU', 'RESOURCE', 'INVITE.ASSET_OWNER', 'DELETE'), - (192, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), - (193, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/agents'), - (194, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges'), - (195, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), - (196, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/space'), - (197, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/market'), - (198, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/models'), - (199, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'CREATE'), - (200, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'READ'), - (201, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'UPDATE'), - (202, 'ASSET_OWNER', 'RESOURCE', 'AGENT', 'DELETE'), - (203, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'CREATE'), - (204, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'READ'), - (205, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'UPDATE'), - (206, 'ASSET_OWNER', 'RESOURCE', 'SKILL', 'DELETE'), - (207, 'ASSET_OWNER', 'RESOURCE', 'KB', 'CREATE'), - (208, 'ASSET_OWNER', 'RESOURCE', 'KB', 'READ'), - (209, 'ASSET_OWNER', 'RESOURCE', 'KB', 'UPDATE'), - (210, 'ASSET_OWNER', 'RESOURCE', 'KB', 'DELETE'), - (211, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'CREATE'), - (212, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'READ'), - (213, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'UPDATE'), - (214, 'ASSET_OWNER', 'RESOURCE', 'MCP', 'DELETE'), - (215, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'CREATE'), - (216, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'READ'), - (217, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'UPDATE'), - (218, 'ASSET_OWNER', 'RESOURCE', 'MODEL', 'DELETE'), - (219, 'ASSET_OWNER', 'RESOURCE', 'USER.ROLE', 'READ'), - (220, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/users'), - (221, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/asset-owner-resources') -ON CONFLICT (role_permission_id) DO NOTHING; - -COMMIT; diff --git a/docker/sql/v2.2.1_0601_add_agent_verification_config.sql b/docker/sql/v2.2.1_0601_add_agent_verification_config.sql deleted file mode 100644 index d3882e1e2..000000000 --- a/docker/sql/v2.2.1_0601_add_agent_verification_config.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Migration: Add layered ReAct self-verification config to agents --- Description: Stores per-agent verification controls for step-level and final-answer validation. - -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS verification_config JSONB; - -COMMENT ON COLUMN nexent.ag_tenant_agent_t.verification_config IS 'Layered ReAct self-verification configuration'; diff --git a/docker/sql/v2.2.1_0601_add_preserve_source_file_to_knowledge_record_t.sql b/docker/sql/v2.2.1_0601_add_preserve_source_file_to_knowledge_record_t.sql deleted file mode 100644 index 30b588a51..000000000 --- a/docker/sql/v2.2.1_0601_add_preserve_source_file_to_knowledge_record_t.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Migration: Add preserve_source_file to knowledge_record_t table --- Date: 2026-06-01 --- Description: Whether to preserve uploaded source documents after vectorization (default: true) - -ALTER TABLE nexent.knowledge_record_t -ADD COLUMN IF NOT EXISTS preserve_source_file BOOLEAN NOT NULL DEFAULT true; - -COMMENT ON COLUMN nexent.knowledge_record_t.preserve_source_file IS 'Whether to preserve uploaded source documents after vectorization'; diff --git a/docker/sql/v2.2.1_0603_add_greeting_fields_to_ag_tenant_agent_t.sql b/docker/sql/v2.2.1_0603_add_greeting_fields_to_ag_tenant_agent_t.sql deleted file mode 100644 index 7786bb902..000000000 --- a/docker/sql/v2.2.1_0603_add_greeting_fields_to_ag_tenant_agent_t.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Migration: Add greeting_message and example_questions columns to ag_tenant_agent_t table --- Date: 2026-06-03 --- Description: Add greeting message and example questions fields for agent chat initial screen - --- Add greeting_message column to ag_tenant_agent_t table -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS greeting_message TEXT; - --- Add example_questions column to ag_tenant_agent_t table -ALTER TABLE nexent.ag_tenant_agent_t -ADD COLUMN IF NOT EXISTS example_questions JSONB; - --- Add comments to the columns -COMMENT ON COLUMN nexent.ag_tenant_agent_t.greeting_message IS 'Agent greeting message displayed on chat initial screen'; -COMMENT ON COLUMN nexent.ag_tenant_agent_t.example_questions IS 'List of example questions for starting a conversation with this agent'; \ No newline at end of file diff --git a/docker/sql/v2.2.1_0605_add_ag_agent_repository_t.sql b/docker/sql/v2.2.1_0605_add_ag_agent_repository_t.sql deleted file mode 100644 index d719fc5aa..000000000 --- a/docker/sql/v2.2.1_0605_add_ag_agent_repository_t.sql +++ /dev/null @@ -1,96 +0,0 @@ --- Migration: Add ag_agent_repository_t table --- Date: 2026-06-05 --- Description: Agent marketplace repository for frozen shareable agent snapshots. - -SET search_path TO nexent; - -BEGIN; - -CREATE SEQUENCE IF NOT EXISTS nexent.ag_agent_repository_t_agent_repository_id_seq; - -CREATE TABLE IF NOT EXISTS nexent.ag_agent_repository_t ( - agent_repository_id BIGINT NOT NULL DEFAULT nextval('nexent.ag_agent_repository_t_agent_repository_id_seq'), - publisher_tenant_id VARCHAR(100) NOT NULL, - publisher_user_id VARCHAR(100) NOT NULL, - agent_id INTEGER NOT NULL, - source_version_no INTEGER NOT NULL, - name VARCHAR(100) NOT NULL, - display_name VARCHAR(100), - description TEXT, - author VARCHAR(100), - category_id INTEGER, - tags TEXT[], - tool_count INTEGER, - version_label VARCHAR(100), - agent_info_json JSONB NOT NULL, - status VARCHAR(30) DEFAULT 'NOT_SHARED', - create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - created_by VARCHAR(100), - updated_by VARCHAR(100), - delete_flag VARCHAR(1) DEFAULT 'N', - CONSTRAINT ag_agent_repository_t_pkey PRIMARY KEY (agent_repository_id) -); - -ALTER SEQUENCE nexent.ag_agent_repository_t_agent_repository_id_seq - OWNED BY nexent.ag_agent_repository_t.agent_repository_id; - -ALTER TABLE nexent.ag_agent_repository_t OWNER TO root; - -COMMENT ON TABLE nexent.ag_agent_repository_t IS 'Agent marketplace repository for frozen shareable agent snapshots'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.agent_repository_id IS 'Agent repository listing ID, unique primary key'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.publisher_tenant_id IS 'Publisher tenant ID'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.publisher_user_id IS 'Publisher user ID'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.agent_id IS 'Root agent ID from ag_tenant_agent_t; upsert key with publisher_tenant_id'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.source_version_no IS 'Published version number frozen at share time'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.name IS 'Root agent programmatic name for display and search'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.display_name IS 'Root agent display name'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.description IS 'Root agent description'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.author IS 'Agent author'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.category_id IS 'Optional marketplace category ID'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.tags IS 'Marketplace tags'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.tool_count IS 'Total tool count across all agents in the bundle (display only)'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.version_label IS 'Repository entry version label for display (e.g. v1.0)'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.agent_info_json IS 'Frozen ExportAndImportDataFormat snapshot with optional skills'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.status IS 'Listing status: NOT_SHARED (未共享) / PENDING_REVIEW (待审核) / REJECTED (审核驳回) / SHARED (已共享)'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.create_time IS 'Creation time'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.update_time IS 'Update time'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.created_by IS 'Creator ID'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.updated_by IS 'Updater ID'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.delete_flag IS 'Soft delete flag: Y/N'; - -CREATE UNIQUE INDEX IF NOT EXISTS uq_agent_repository_tenant_agent_active - ON nexent.ag_agent_repository_t (publisher_tenant_id, agent_id) - WHERE delete_flag = 'N'; - -CREATE INDEX IF NOT EXISTS idx_agent_repository_publisher_delete - ON nexent.ag_agent_repository_t (publisher_tenant_id, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_agent_repository_status_delete - ON nexent.ag_agent_repository_t (status, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_agent_repository_name_delete - ON nexent.ag_agent_repository_t (name, delete_flag); - -CREATE INDEX IF NOT EXISTS idx_agent_repository_tags_gin - ON nexent.ag_agent_repository_t USING GIN (tags); - -CREATE OR REPLACE FUNCTION update_ag_agent_repository_update_time() -RETURNS TRIGGER AS $$ -BEGIN - NEW.update_time = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -COMMENT ON FUNCTION update_ag_agent_repository_update_time() IS 'Auto-update update_time for ag_agent_repository_t'; - -DROP TRIGGER IF EXISTS update_ag_agent_repository_update_time_trigger ON nexent.ag_agent_repository_t; -CREATE TRIGGER update_ag_agent_repository_update_time_trigger -BEFORE UPDATE ON nexent.ag_agent_repository_t -FOR EACH ROW -EXECUTE FUNCTION update_ag_agent_repository_update_time(); - -COMMENT ON TRIGGER update_ag_agent_repository_update_time_trigger ON nexent.ag_agent_repository_t IS 'Trigger to maintain update_time'; - -COMMIT; diff --git a/docker/sql/v2.2.1_0609_add_selected_agent_version_no_to_agent_relation_t.sql b/docker/sql/v2.2.1_0609_add_selected_agent_version_no_to_agent_relation_t.sql deleted file mode 100644 index 9a67c1ab2..000000000 --- a/docker/sql/v2.2.1_0609_add_selected_agent_version_no_to_agent_relation_t.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Migration: Add selected_agent_version_no to ag_agent_relation_t --- Date: 2026-06-09 --- Description: Pin child agent version on parent-child relations at publish time. - -SET search_path TO nexent; - -BEGIN; - -ALTER TABLE nexent.ag_agent_relation_t - ADD COLUMN IF NOT EXISTS selected_agent_version_no INTEGER; - -COMMENT ON COLUMN nexent.ag_agent_relation_t.selected_agent_version_no IS - 'Pinned version of selected_agent_id. NULL = use child current published version at runtime (legacy/draft).'; - -COMMIT; diff --git a/docker/upgrade.sh b/docker/upgrade.sh deleted file mode 100644 index 38684dae0..000000000 --- a/docker/upgrade.sh +++ /dev/null @@ -1,420 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -OPTIONS_FILE="$SCRIPT_DIR/deploy.options" -CONST_FILE="$PROJECT_ROOT/backend/consts/const.py" -DEPLOY_SCRIPT="$SCRIPT_DIR/deploy.sh" -SQL_DIR="$SCRIPT_DIR/sql" -ENV_FILE="$SCRIPT_DIR/.env" -V180_SCRIPT="$SCRIPT_DIR/scripts/v180_sync_user_metadata.sh" -V180_VERSION="1.8.0" - -declare -A DEPLOY_OPTIONS -UPGRADE_SQL_FILES=() - -log() { - local level="$1" - shift - printf "[%s] %s\n" "$level" "$*" -} - -require_file() { - local path="$1" - local message="$2" - if [ ! -f "$path" ]; then - log "ERROR" "$message" - exit 1 - fi -} - -trim_quotes() { - local value="$1" - value="${value%$'\r'}" - value="${value%\"}" - value="${value#\"}" - echo "$value" -} - -load_options() { - if [ ! -f "$OPTIONS_FILE" ]; then - log "WARN" "⚙️ deploy.options not found, entering interactive configuration mode." - : > "$OPTIONS_FILE" - return - fi - while IFS= read -r line || [ -n "$line" ]; do - [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue - if [[ "$line" =~ ^[[:space:]]*([A-Za-z0-9_]+)[[:space:]]*=(.*)$ ]]; then - local key="${BASH_REMATCH[1]}" - local raw_value="${BASH_REMATCH[2]}" - raw_value="$(echo "$raw_value" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" - DEPLOY_OPTIONS[$key]="$(trim_quotes "$raw_value")" - fi - done < "$OPTIONS_FILE" -} - -prompt_option_value() { - local key="$1" - local prompt_msg="$2" - local default_value="${3:-}" - local input_type="${4:-text}" # Default to text type - local input="" - - while true; do - read -rp "${prompt_msg}: " input - - input="$(trim_quotes "$input")" - - # Handle yes/no type inputs - if [[ "$input_type" == "boolean" ]]; then - # Convert to uppercase for consistency - input=$(echo "$input" | tr '[:lower:]' '[:upper:]') - - # Validate input - if [[ "$input" =~ ^[YN]$ ]]; then - DEPLOY_OPTIONS[$key]="$input" - update_option_value "$key" "$input" - break - elif [ -z "$input" ] && [ -n "$default_value" ]; then - # Use default value if input is empty - DEPLOY_OPTIONS[$key]="$default_value" - update_option_value "$key" "$default_value" - break - fi - else - # Handle other types of inputs - if [ -n "$input" ]; then - DEPLOY_OPTIONS[$key]="$input" - update_option_value "$key" "$input" - break - elif [ -z "$input" ] && [ -n "$default_value" ]; then - # Use default value if input is empty - DEPLOY_OPTIONS[$key]="$default_value" - update_option_value "$key" "$default_value" - break - fi - fi - - log "WARN" "⚠️ ${key} cannot be empty, please enter a value." - done -} - -require_option() { - local key="$1" - local prompt_msg="${2:-}" - local value="${DEPLOY_OPTIONS[$key]:-}" - if [ -z "$value" ]; then - if [ -n "$prompt_msg" ]; then - prompt_option_value "$key" "$prompt_msg" - else - log "ERROR" "❌ ${key} is missing in deploy.options, add it and rerun." - exit 1 - fi - fi -} - -get_const_app_version() { - require_file "$CONST_FILE" "backend/consts/const.py not found, unable to read the latest version." - local line - line=$(grep -E 'APP_VERSION' "$CONST_FILE" | tail -n 1 || true) - line="${line##*=}" - line="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" - trim_quotes "$line" -} - -compare_versions() { - local v1="${1#v}" - local v2="${2#v}" - IFS='.' read -r -a parts1 <<< "$v1" - IFS='.' read -r -a parts2 <<< "$v2" - local max_len="${#parts1[@]}" - if [ "${#parts2[@]}" -gt "$max_len" ]; then - max_len="${#parts2[@]}" - fi - for ((i=0; i 10#$num2)) && { echo 1; return; } - ((10#$num1 < 10#$num2)) && { echo -1; return; } - done - echo 0 -} - -collect_upgrade_sqls() { - if [ ! -d "$SQL_DIR" ]; then - log "WARN" "📭 SQL directory not found, skipping database upgrade scripts." - return - fi - - mapfile -t sql_files < <(find "$SQL_DIR" -maxdepth 1 -type f -name "v*.sql" -print | sort -V || true) - if [ "${#sql_files[@]}" -eq 0 ]; then - return - fi - - for file in "${sql_files[@]}"; do - local base version_prefix - base="$(basename "$file")" - version_prefix="${base%%_*}" - [[ -z "$version_prefix" ]] && continue - - local cmp_current - cmp_current="$(compare_versions "$version_prefix" "$CURRENT_APP_VERSION")" - - if [ "$cmp_current" -eq 1 ]; then - UPGRADE_SQL_FILES+=("$file") - fi - done -} - -build_deploy_args() { - DEPLOY_ARGS=() - local mode="${DEPLOY_OPTIONS[MODE_CHOICE]:-}" - local version_choice="${DEPLOY_OPTIONS[VERSION_CHOICE]:-}" - local is_mainland="${DEPLOY_OPTIONS[IS_MAINLAND]:-}" - local enable_terminal="${DEPLOY_OPTIONS[ENABLE_TERMINAL]:-}" - local root_dir="${DEPLOY_OPTIONS[ROOT_DIR]:-}" - - [[ -n "$mode" ]] && DEPLOY_ARGS+=(--mode "$mode") - [[ -n "$version_choice" ]] && DEPLOY_ARGS+=(--version "$version_choice") - [[ -n "$is_mainland" ]] && DEPLOY_ARGS+=(--is-mainland "$is_mainland") - [[ -n "$enable_terminal" ]] && DEPLOY_ARGS+=(--enable-terminal "$enable_terminal") - [[ -n "$root_dir" ]] && DEPLOY_ARGS+=(--root-dir "$root_dir") -} - -ensure_docker() { - if ! command -v docker >/dev/null 2>&1; then - log "ERROR" "🛑 Docker CLI not detected, install Docker before continuing." - exit 1 - fi -} - -ensure_postgres_env() { - require_file "$ENV_FILE" "📁 docker/.env not found; unable to load database credentials." - set -a - source "$ENV_FILE" - set +a - : "${POSTGRES_USER:?docker/.env is missing POSTGRES_USER}" - : "${POSTGRES_DB:?docker/.env is missing POSTGRES_DB}" -} - -run_deploy() { - # Stop and remove any existing containers before redeployment - docker compose -p nexent down -v - log "INFO" "🚀 Starting deploy..." - (cd "$SCRIPT_DIR" && cp .env.example .env && bash "$DEPLOY_SCRIPT" "${DEPLOY_ARGS[@]}") - -} - -run_sql_scripts() { - if [ "${#UPGRADE_SQL_FILES[@]}" -eq 0 ]; then - log "INFO" "📭 No database upgrade scripts detected, skipping this step." - return - fi - - ensure_postgres_env - - for sql_file in "${UPGRADE_SQL_FILES[@]}"; do - log "INFO" "🗃️ Running database upgrade script $(basename "$sql_file") ..." - if ! docker exec -i nexent-postgresql psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 < "$sql_file"; then - log "ERROR" "❌ Failed to execute $(basename "$sql_file"), please verify the script." - exit 1 - fi - done -} - -update_option_value() { - local key="$1" - local value="$2" - touch "$OPTIONS_FILE" - if grep -q "^${key}[[:space:]]*=" "$OPTIONS_FILE"; then - sed -i.bak -E "s|^(${key}[[:space:]]*=[[:space:]]*)\"?[^\"]*\"?|\1\"${value}\"|" "$OPTIONS_FILE" - else - echo "${key} = \"${value}\"" >> "$OPTIONS_FILE" - fi -} - -# Check if the upgrade version span includes v1.8.0 -# Returns 0 (success) if span includes v1.8.0, 1 otherwise -check_version_spans_v180() { - local cmp_with_v180 - local cmp_current - - # Check if current version is less than v1.8.0 - cmp_current="$(compare_versions "$CURRENT_APP_VERSION" "$V180_VERSION")" - if [ "$cmp_current" -ge 0 ]; then - # Current version is >= v1.8.0, no need to run v180 sync - return 1 - fi - - # Check if target version is >= v1.8.0 - cmp_with_v180="$(compare_versions "$NEW_APP_VERSION" "$V180_VERSION")" - if [ "$cmp_with_v180" -lt 0 ]; then - # Target version is < v1.8.0, no need to run v180 sync - return 1 - fi - - # Version span includes v1.8.0 - return 0 -} - -# Execute the v1.8.0 user metadata sync script -run_v180_sync_script() { - if [ ! -f "$V180_SCRIPT" ]; then - log "WARN" "⚠️ v180_sync_user_metadata.sh not found, skipping v1.8.0 metadata sync." - return - fi - - log "INFO" "🗄️ Detected version span includes v1.8.0, executing user metadata sync script..." - - if ! bash "$V180_SCRIPT"; then - log "ERROR" "❌ Failed to execute v180_sync_user_metadata.sh, please verify the script." - exit 1 - fi - - log "INFO" "✅ v1.8.0 user metadata sync completed successfully." -} - - -prompt_deploy_options() { - # Only prompt for options that already exist in DEPLOY_OPTIONS - if [[ -n "${DEPLOY_OPTIONS[VERSION_CHOICE]:-}" ]]; then - echo "🚀 Please select deployment version:" - echo " 1) ⚡️ Speed version - Lightweight deployment with essential features" - echo " 2) 🎯 Full version - Full-featured deployment with all capabilities" - prompt_option_value "VERSION_CHOICE" "Enter your choice [1/2] (default: ${DEPLOY_OPTIONS[VERSION_CHOICE]:-1})" "${DEPLOY_OPTIONS[VERSION_CHOICE]:-1}" "text" - fi - if [[ -n "${DEPLOY_OPTIONS[MODE_CHOICE]:-}" ]]; then - echo "🎛️ Please select deployment mode:" - echo " 1) 🛠️ Development mode - Expose all service ports for debugging" - echo " 2) 🏗️ Infrastructure mode - Only start infrastructure services" - echo " 3) 🚀 Production mode - Only expose port 3000 for security" - prompt_option_value "MODE_CHOICE" "Enter your choice [1/2/3] (default: ${DEPLOY_OPTIONS[MODE_CHOICE]:-1})" "${DEPLOY_OPTIONS[MODE_CHOICE]:-1}" "text" - fi - if [[ -n "${DEPLOY_OPTIONS[ENABLE_TERMINAL]:-}" ]]; then - prompt_option_value "ENABLE_TERMINAL" "Do you want to create Terminal tool container? [Y/N] (default: ${DEPLOY_OPTIONS[ENABLE_TERMINAL]:-N})" "${DEPLOY_OPTIONS[ENABLE_TERMINAL]:-N}" "boolean" - fi - if [[ -n "${DEPLOY_OPTIONS[IS_MAINLAND]:-}" ]]; then - prompt_option_value "IS_MAINLAND" "Is your server network located in mainland China? [Y/N] (default: ${DEPLOY_OPTIONS[IS_MAINLAND]:-N})" "${DEPLOY_OPTIONS[IS_MAINLAND]:-N}" "boolean" - fi -} - -# Get friendly description for option keys -_get_option_description() { - local key="$1" - case "$key" in - "MODE_CHOICE") echo "Deployment Mode" ;; - "VERSION_CHOICE") echo "Deployment Version" ;; - "IS_MAINLAND") echo "Mainland China Network" ;; - "ENABLE_TERMINAL") echo "Terminal Tool Container" ;; - "APP_VERSION") echo "Application Version" ;; - "ROOT_DIR") echo "Root Directory" ;; - *) echo "$key" ;; - esac -} - -# Get friendly value for option values -_get_option_value_description() { - local key="$1" - local value="$2" - - case "$key" in - "MODE_CHOICE") - case "$value" in - "1") echo "1 - Development Mode" ;; - "2") echo "2 - Infrastructure Mode" ;; - "3") echo "3 - Production Mode" ;; - *) echo "$value" ;; - esac - ;; - "VERSION_CHOICE") - case "$value" in - "1") echo "1 - Speed Version" ;; - "2") echo "2 - Full Version" ;; - *) echo "$value" ;; - esac - ;; - *) echo "$value" ;; - esac -} - -main() { - ensure_docker - load_options - - # Ensure required options are present - require_option "APP_VERSION" "APP_VERSION not detected, please enter the current deployed version" - require_option "ROOT_DIR" "ROOT_DIR not detected, please enter the absolute deployment directory path" - CURRENT_APP_VERSION="${DEPLOY_OPTIONS[APP_VERSION]:-}" - - NEW_APP_VERSION="$(get_const_app_version)" - if [ -z "$NEW_APP_VERSION" ]; then - log "ERROR" "❌ Unable to parse APP_VERSION from const.py, please verify the file." - exit 1 - fi - - log "INFO" "📦 Current version: $CURRENT_APP_VERSION" - log "INFO" "🎯 Target version: $NEW_APP_VERSION" - - local cmp_result - cmp_result="$(compare_versions "$NEW_APP_VERSION" "$CURRENT_APP_VERSION")" - if [ "$cmp_result" -le 0 ]; then - log "INFO" "🚫 Target version ($NEW_APP_VERSION) is not higher than current version ($CURRENT_APP_VERSION), upgrade aborted." - exit 1 - fi - - # Ask user if they want to inherit previous deployment options - if [ -f "$OPTIONS_FILE" ] && [ -s "$OPTIONS_FILE" ]; then - # Calculate maximum width of option descriptions for better alignment - max_desc_width=0 - for key in "${!DEPLOY_OPTIONS[@]}"; do - desc=$(_get_option_description "$key") - desc_length=${#desc} - if (( desc_length > max_desc_width )); then - max_desc_width=$desc_length - fi - done - - # Ensure minimum width for better readability - if (( max_desc_width < 20 )); then - max_desc_width=20 - fi - - # Display current deployment options in a readable format - log "INFO" "📋 Current deployment options:" - echo "" - for key in "${!DEPLOY_OPTIONS[@]}"; do - value="${DEPLOY_OPTIONS[$key]}" - desc=$(_get_option_description "$key") - value_desc=$(_get_option_value_description "$key" "$value") - printf " • %-${max_desc_width}s : %s\n" "$desc" "$value_desc" - done - echo "" - - read -rp "🔄 Do you want to inherit previous deployment options? [Y/N] (default: Y): " inherit_choice - inherit_choice="${inherit_choice:-Y}" - inherit_choice="$(trim_quotes "$inherit_choice")" - if [[ "$inherit_choice" =~ ^[Nn]$ ]]; then - log "INFO" "📝 Starting configuration..." - # Prompt for deployment options with existing values as defaults - prompt_deploy_options - fi - fi - - build_deploy_args - run_deploy - - # Check if version span includes v1.8.0 and run sync script if needed - if check_version_spans_v180; then - run_v180_sync_script - fi - - collect_upgrade_sqls - run_sql_scripts - - log "INFO" "🎉 Upgrade to ${NEW_APP_VERSION} completed, please verify service health." -} - -main "$@" - diff --git a/docker/volumes/logs/vector.yml b/docker/volumes/logs/vector.yml deleted file mode 100644 index cce46df43..000000000 --- a/docker/volumes/logs/vector.yml +++ /dev/null @@ -1,232 +0,0 @@ -api: - enabled: true - address: 0.0.0.0:9001 - -sources: - docker_host: - type: docker_logs - exclude_containers: - - supabase-vector - -transforms: - project_logs: - type: remap - inputs: - - docker_host - source: |- - .project = "default" - .event_message = del(.message) - .appname = del(.container_name) - del(.container_created_at) - del(.container_id) - del(.source_type) - del(.stream) - del(.label) - del(.image) - del(.host) - del(.stream) - router: - type: route - inputs: - - project_logs - route: - kong: '.appname == "supabase-kong"' - auth: '.appname == "supabase-auth"' - rest: '.appname == "supabase-rest"' - realtime: '.appname == "supabase-realtime"' - storage: '.appname == "supabase-storage"' - functions: '.appname == "supabase-functions"' - db: '.appname == "supabase-db"' - # Ignores non nginx errors since they are related with kong booting up - kong_logs: - type: remap - inputs: - - router.kong - source: |- - req, err = parse_nginx_log(.event_message, "combined") - if err == null { - .timestamp = req.timestamp - .metadata.request.headers.referer = req.referer - .metadata.request.headers.user_agent = req.agent - .metadata.request.headers.cf_connecting_ip = req.client - .metadata.request.method = req.method - .metadata.request.path = req.path - .metadata.request.protocol = req.protocol - .metadata.response.status_code = req.status - } - if err != null { - abort - } - # Ignores non nginx errors since they are related with kong booting up - kong_err: - type: remap - inputs: - - router.kong - source: |- - .metadata.request.method = "GET" - .metadata.response.status_code = 200 - parsed, err = parse_nginx_log(.event_message, "error") - if err == null { - .timestamp = parsed.timestamp - .severity = parsed.severity - .metadata.request.host = parsed.host - .metadata.request.headers.cf_connecting_ip = parsed.client - url, err = split(parsed.request, " ") - if err == null { - .metadata.request.method = url[0] - .metadata.request.path = url[1] - .metadata.request.protocol = url[2] - } - } - if err != null { - abort - } - # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency. - auth_logs: - type: remap - inputs: - - router.auth - source: |- - parsed, err = parse_json(.event_message) - if err == null { - .metadata.timestamp = parsed.time - .metadata = merge!(.metadata, parsed) - } - # PostgREST logs are structured so we separate timestamp from message using regex - rest_logs: - type: remap - inputs: - - router.rest - source: |- - parsed, err = parse_regex(.event_message, r'^(?P